10:29:29.554 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
10:29:29.554 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskM5Ix5b/test-tmp
10:29:29.554 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
10:29:29.555 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskM5Ix5b/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
10:29:29.556 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
10:29:29.877 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:29:29.877 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:29:29.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:29.880589 10305 mini_oidc.cc:150] Starting JWKS server
10:29:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:29.883782 10305 webserver.cc:458] Webserver started at http://127.0.0.1:36871/ using document root <none> and password file <none>
10:29:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:29.884219 10305 mini_oidc.cc:186] Starting OIDC Discovery server
10:29:29.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:29.885874 10305 webserver.cc:458] Webserver started at http://127.0.0.1:45277/ using document root <none> and password file <none>
10:29:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:29:29Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
10:29:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:29:29Z Disabled control of system clock
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:29.947000 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/logs
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/info.pb
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.126:37021
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.126
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:29:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:29:30.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.250110 10317 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.250733 10317 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.251247 10317 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.282677 10317 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:29:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.283002 10317 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:29:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.283164 10317 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.283375 10317 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:29:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.283576 10317 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:29:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.283767 10317 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:29:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.283919 10317 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36871/jwks.json
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:30.320899 10317 master_runner.cc:386] Master server non-default flags:
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.126:37021
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/info.pb
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.126
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/logs
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:30.322219 10317 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:30.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:30.323772 10317 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.339107 10323 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.339241 10324 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:30.341248 10317 server_base.cc:1034] running on GCE node
10:29:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:30.340504 10326 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.516870 10317 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.519414 10317 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.520743 10317 hybrid_clock.cc:648] HybridClock initialized: now 1736936971520702 us; error 57 us; skew 500 ppm
10:29:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.521509 10317 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.533829 10317 webserver.cc:458] Webserver started at http://127.10.16.126:32917/ using document root <none> and password file <none>
10:29:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.534674 10317 fs_manager.cc:362] Metadata directory not provided
10:29:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.534852 10317 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.535255 10317 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:29:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.540078 10317 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/instance:
10:29:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2c6a9ec095f4488875263c4c31e1963"
10:29:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:31 on dist-test-slave-gn4w"
10:29:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.541134 10317 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal/instance:
10:29:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2c6a9ec095f4488875263c4c31e1963"
10:29:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:31 on dist-test-slave-gn4w"
10:29:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.550823 10317 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
10:29:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.556495 10333 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:31.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.557724 10317 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.558025 10317 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2c6a9ec095f4488875263c4c31e1963"
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:31 on dist-test-slave-gn4w"
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.558336 10317 fs_report.cc:389] FS layout report
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/data
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.625276 10317 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:31.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.627214 10317 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:31.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.627694 10317 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.708636 10317 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.126:37021
10:29:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.708724 10394 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.126:37021 every 8 connection(s)
10:29:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.711279 10317 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/info.pb
10:29:31.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.716213 10395 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:31.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.721120 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 10317
10:29:31.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.721482 10305 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal/instance
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.724996 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/logs
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data/info.pb
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.125:40145
10:29:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.125
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:29:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:29:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.727735 10395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } has no permanent_uuid. Determining permanent_uuid...
10:29:31.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.753584 10395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } has no permanent_uuid. Determining permanent_uuid...
10:29:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:31.756255 10334 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:31.760506 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:31.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.809991 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 1
10:29:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:31.814903 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.883302 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 2
10:29:31.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:31.887738 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:31.979171 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 3
10:29:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:31.983409 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.045568 10397 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.046487 10397 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:32.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.047435 10397 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.085222 10397 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:29:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.085511 10397 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:29:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.085668 10397 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.085867 10397 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:29:32.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.086056 10397 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:29:32.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.086242 10397 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:29:32.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.086385 10397 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36871/jwks.json
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.122664 10397 master_runner.cc:386] Master server non-default flags:
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.125:40145
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data/info.pb
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.125
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/logs
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:32.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.123960 10397 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.125553 10397 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:32.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.126797 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 4
10:29:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.130960 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.142325 10406 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.430405 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 5
10:29:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.436160 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.983572 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 6
10:29:32.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.990319 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:33.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.143126 10409 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:33.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:32.144055 10397 server_base.cc:1034] running on GCE node
10:29:33.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:32.144166 10407 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.317322 10397 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.320343 10397 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.321779 10397 hybrid_clock.cc:648] HybridClock initialized: now 1736936973321721 us; error 71 us; skew 500 ppm
10:29:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.322546 10397 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:33.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.330986 10397 webserver.cc:458] Webserver started at http://127.10.16.125:43507/ using document root <none> and password file <none>
10:29:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.332234 10397 fs_manager.cc:362] Metadata directory not provided
10:29:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.332494 10397 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.332957 10397 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:29:33.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.339110 10397 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data/instance:
10:29:33.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53d1bd87fb0148c780618938c1137c6c"
10:29:33.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:33 on dist-test-slave-gn4w"
10:29:33.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.340626 10397 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal/instance:
10:29:33.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53d1bd87fb0148c780618938c1137c6c"
10:29:33.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:33 on dist-test-slave-gn4w"
10:29:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.349144 10397 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
10:29:33.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.355741 10418 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.356959 10397 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
10:29:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.357375 10397 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal
10:29:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53d1bd87fb0148c780618938c1137c6c"
10:29:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:33 on dist-test-slave-gn4w"
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.357817 10397 fs_report.cc:389] FS layout report
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data/data
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:33.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.408259 10397 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.409579 10397 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:33.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.410028 10397 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.487905 10397 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.125:40145
10:29:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.488010 10479 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.125:40145 every 8 connection(s)
10:29:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.490722 10397 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/data/info.pb
10:29:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.495963 10480 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.498978 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 10397
10:29:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.499423 10305 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-1/wal/instance
10:29:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.501669 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/logs
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/info.pb
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.124:34587
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.124
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:29:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:29:33.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.506646 10480 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } has no permanent_uuid. Determining permanent_uuid...
10:29:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.527035 10480 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } has no permanent_uuid. Determining permanent_uuid...
10:29:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.541213 10480 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } has no permanent_uuid. Determining permanent_uuid...
10:29:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.547044 10421 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.552947 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.602519 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 1
10:29:33.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.606196 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.674640 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 2
10:29:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.678270 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.769685 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 3
10:29:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.773427 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.812062 10482 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.812651 10482 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:33.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.813120 10482 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.843196 10482 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:29:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.843461 10482 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:29:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.843608 10482 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.843775 10482 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:29:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.843958 10482 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:29:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.844148 10482 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:29:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.844273 10482 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36871/jwks.json
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.876401 10482 master_runner.cc:386] Master server non-default flags:
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:29:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.124:34587
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/info.pb
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.124
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/logs
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.877597 10482 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.878984 10482 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:33.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.893224 10492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.916882 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 4
10:29:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.921196 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.050969 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } attempt: 7
10:29:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.066723 10395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } has no permanent_uuid. Determining permanent_uuid...
10:29:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.072077 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.130674 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 1
10:29:34.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.134974 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.216408 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 2
10:29:34.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.220208 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.220633 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 5
10:29:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.224042 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.305840 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 3
10:29:34.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.310050 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.450693 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 4
10:29:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.454893 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.738490 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 5
10:29:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.742350 10395 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:34.771688 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 6
10:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:34.779546 10480 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:35.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.894033 10493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:35.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:33.895278 10482 server_base.cc:1034] running on GCE node
10:29:35.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:33.894464 10495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:35.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.062613 10482 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.065686 10482 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.067078 10482 hybrid_clock.cc:648] HybridClock initialized: now 1736936975067034 us; error 55 us; skew 500 ppm
10:29:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.067835 10482 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.074573 10482 webserver.cc:458] Webserver started at http://127.10.16.124:33155/ using document root <none> and password file <none>
10:29:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.075456 10482 fs_manager.cc:362] Metadata directory not provided
10:29:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.075647 10482 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:35.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.076026 10482 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:29:35.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.080761 10482 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/instance:
10:29:35.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b21963161d48bfa33fa722af5c6c36"
10:29:35.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:35 on dist-test-slave-gn4w"
10:29:35.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.081769 10482 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal/instance:
10:29:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b21963161d48bfa33fa722af5c6c36"
10:29:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:35 on dist-test-slave-gn4w"
10:29:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.088784 10482 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.004s
10:29:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.095052 10507 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.096243 10482 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.001s
10:29:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.096637 10482 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b21963161d48bfa33fa722af5c6c36"
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:35 on dist-test-slave-gn4w"
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.096951 10482 fs_report.cc:389] FS layout report
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/data
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:35.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.154124 10482 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.155525 10482 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.155915 10482 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:35.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.238131 10482 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.124:34587
10:29:35.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.238214 10568 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.124:34587 every 8 connection(s)
10:29:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.240722 10482 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/info.pb
10:29:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.242122 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 10482
10:29:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.242506 10305 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal/instance
10:29:35.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.248344 10569 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.264231 10569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } has no permanent_uuid. Determining permanent_uuid...
10:29:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.285028 10569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } has no permanent_uuid. Determining permanent_uuid...
10:29:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.294724 10395 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 6
10:29:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.295006 10569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } has no permanent_uuid. Determining permanent_uuid...
10:29:35.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.313874 10395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Bootstrap starting.
10:29:35.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.313943 10569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Bootstrap starting.
10:29:35.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.319456 10395 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Neither blocks nor log segments found. Creating new log.
10:29:35.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.319591 10569 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Neither blocks nor log segments found. Creating new log.
10:29:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.321183 10395 log.cc:826] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Log is configured to *not* fsync() on all Append() calls
10:29:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.321405 10569 log.cc:826] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Log is configured to *not* fsync() on all Append() calls
10:29:35.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.326146 10395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: No bootstrap required, opened a new log
10:29:35.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.326527 10569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: No bootstrap required, opened a new log
10:29:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.348713 10569 raft_consensus.cc:357] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.348737 10395 raft_consensus.cc:357] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.349406 10569 raft_consensus.cc:383] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.349465 10395 raft_consensus.cc:383] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.349680 10395 raft_consensus.cc:738] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2c6a9ec095f4488875263c4c31e1963, State: Initialized, Role: FOLLOWER
10:29:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.349699 10569 raft_consensus.cc:738] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 39b21963161d48bfa33fa722af5c6c36, State: Initialized, Role: FOLLOWER
10:29:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.350334 10395 consensus_queue.cc:260] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.350469 10569 consensus_queue.cc:260] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.352877 10580 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.353878 10580 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: FOLLOWER
10:29:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.353425 10581 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.354283 10581 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.354849 10395 sys_catalog.cc:564] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: configured and running, proceeding with master startup.
10:29:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.355144 10569 sys_catalog.cc:564] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: configured and running, proceeding with master startup.
10:29:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:35.386708 10339 tablet.cc:2367] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:29:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:35.407616 10603 catalog_manager.cc:1559] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:29:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:35.407971 10603 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:29:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:35.407955 10600 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:29:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:35.408353 10600 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:29:35.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:35.411056 10513 tablet.cc:2367] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:29:35.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.420567 10580 raft_consensus.cc:491] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:29:35.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.421149 10580 raft_consensus.cc:513] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.386222 10482 master_runner.cc:186] Error getting master registration for 127.10.16.126:37021: OK, instance_id { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" instance_seqno: 1736936971690255 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:29:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.425956 10534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "39b21963161d48bfa33fa722af5c6c36" is_pre_election: true
10:29:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.426848 10534 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2c6a9ec095f4488875263c4c31e1963 in term 0.
10:29:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.428479 10336 leader_election.cc:304] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39b21963161d48bfa33fa722af5c6c36, a2c6a9ec095f4488875263c4c31e1963; no voters:
10:29:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.393127 10317 master_runner.cc:186] Error getting master registration for 127.10.16.125:40145: OK, instance_id { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" instance_seqno: 1736936973469135 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:29:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.432053 10317 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:29:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.433058 10580 leader_election.cc:290] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.433584 10580 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:29:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.433908 10580 raft_consensus.cc:491] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:29:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.434103 10482 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:29:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.434237 10580 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 0 FOLLOWER]: Advancing to term 1
10:29:35.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.442227 10580 raft_consensus.cc:513] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.444312 10580 leader_election.cc:290] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 1 election: Requested vote from peers 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.446030 10534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "39b21963161d48bfa33fa722af5c6c36"
10:29:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.446628 10534 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 0 FOLLOWER]: Advancing to term 1
10:29:35.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.454604 10534 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2c6a9ec095f4488875263c4c31e1963 in term 1.
10:29:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.455829 10336 leader_election.cc:304] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39b21963161d48bfa33fa722af5c6c36, a2c6a9ec095f4488875263c4c31e1963; no voters:
10:29:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.456947 10580 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 FOLLOWER]: Leader election won for term 1
10:29:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.457507 10580 raft_consensus.cc:695] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 LEADER]: Becoming Leader. State: Replica: a2c6a9ec095f4488875263c4c31e1963, State: Running, Role: LEADER
10:29:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.458498 10580 consensus_queue.cc:237] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.472951 10606 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2c6a9ec095f4488875263c4c31e1963. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.473814 10606 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: LEADER
10:29:35.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.477933 10608 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:29:35.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.515919 10608 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:29:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.571705 10606 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [LEADER]: Connected to new peer: Peer: permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.613087 10581 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2c6a9ec095f4488875263c4c31e1963. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.613878 10581 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.617170 10580 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.617919 10580 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: LEADER
10:29:35.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.619748 10608 catalog_manager.cc:1348] Generated new cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:35.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.620754 10608 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:29:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.623186 10613 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.623983 10613 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.707463 10608 catalog_manager.cc:1371] Generated new certificate authority record
10:29:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.709441 10608 catalog_manager.cc:1505] Loading token signing keys...
10:29:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.744761 10608 catalog_manager.cc:5899] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Generated new TSK 0
10:29:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.745805 10608 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:29:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.772500 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/logs
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data/info.pb
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.65:0
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.16.65
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.65
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:29:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:29:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.839910 10480 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } attempt: 7
10:29:35.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:35.995985 10480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: Bootstrap starting.
10:29:36.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.054301 10480 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: Neither blocks nor log segments found. Creating new log.
10:29:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.072551 10480 log.cc:826] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: Log is configured to *not* fsync() on all Append() calls
10:29:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.090723 10334 leader_election.cc:336] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Timed out: RequestConsensusVote RPC to 127.10.16.125:40145 timed out after 0.300s (SENT)
10:29:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.104077 10480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: No bootstrap required, opened a new log
10:29:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.262143 10616 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.262984 10616 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.264050 10616 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:36.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.279155 10334 leader_election.cc:336] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Timed out: RequestConsensusVote RPC to 127.10.16.125:40145 timed out after 0.419s (SENT)
10:29:36.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.321429 10616 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:36.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.322993 10616 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:29:36.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.323418 10616 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:29:36.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.323763 10616 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:29:36.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.324586 10616 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.16.65
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.377527 10480 raft_consensus.cc:357] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.387893 10616 tablet_server_runner.cc:78] Tablet server non-default flags:
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.65:0
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data/info.pb
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.65
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:29:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.16.65
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/logs
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.390952 10616 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:36.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.393203 10616 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:36.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.389791 10480 raft_consensus.cc:383] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:36.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.394546 10480 raft_consensus.cc:738] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53d1bd87fb0148c780618938c1137c6c, State: Initialized, Role: FOLLOWER
10:29:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.414307 10600 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.414741 10600 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: loading cluster ID for follower catalog manager: success
10:29:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.422286 10600 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: acquiring CA information for follower catalog manager: success
10:29:36.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.429531 10600 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:29:36.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.438820 10480 consensus_queue.cc:260] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:36.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.445433 10627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.447456 10625 raft_consensus.cc:491] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:29:36.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.452651 10625 raft_consensus.cc:513] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.483109 10629 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.483934 10629 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: FOLLOWER
10:29:36.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.518285 10442 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53d1bd87fb0148c780618938c1137c6c"
10:29:36.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.519078 10442 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 0 FOLLOWER]: Advancing to term 1
10:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.524667 10444 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53d1bd87fb0148c780618938c1137c6c"
10:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.525709 10444 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:38370 (request call id 3) took 1081 ms (client timeout 300 ms). Trace:
10:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.526048 10444 rpcz_store.cc:269] 0115 10:29:35.444726 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:35.444835 (+ 109us) service_pool.cc:225] Handling call
10:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:36.525622 (+1080787us) inbound_call.cc:173] Queueing success response
10:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:29:36.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.528184 10445 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53d1bd87fb0148c780618938c1137c6c" is_pre_election: true
10:29:36.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.529017 10445 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:38370 (request call id 2) took 1103 ms (client timeout 419 ms). Trace:
10:29:36.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.529346 10445 rpcz_store.cc:269] 0115 10:29:35.425349 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:29:36.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:35.425509 (+ 160us) service_pool.cc:225] Handling call
10:29:36.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:36.528963 (+1103454us) inbound_call.cc:173] Queueing success response
10:29:36.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:29:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.530776 10441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53d1bd87fb0148c780618938c1137c6c" is_pre_election: true
10:29:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.531580 10441 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:38370 (request call id 6) took 671 ms (client timeout 419 ms). Trace:
10:29:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.531908 10441 rpcz_store.cc:269] 0115 10:29:35.859853 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:29:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:35.860007 (+ 154us) service_pool.cc:225] Handling call
10:29:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:36.531528 (+671521us) inbound_call.cc:173] Queueing success response
10:29:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:29:36.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.534932 10442 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2c6a9ec095f4488875263c4c31e1963 in term 1.
10:29:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.540947 10580 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:36.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.550837 10442 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:38370 (request call id 5) took 760 ms (client timeout 300 ms). Trace:
10:29:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.551551 10442 rpcz_store.cc:269] 0115 10:29:35.790273 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:29:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:35.790404 (+ 131us) service_pool.cc:225] Handling call
10:29:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:36.550773 (+760369us) inbound_call.cc:173] Queueing success response
10:29:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:29:36.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.557473 10360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "53d1bd87fb0148c780618938c1137c6c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2c6a9ec095f4488875263c4c31e1963" is_pre_election: true
10:29:36.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.559115 10480 sys_catalog.cc:564] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: configured and running, proceeding with master startup.
10:29:36.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.563433 10534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "53d1bd87fb0148c780618938c1137c6c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "39b21963161d48bfa33fa722af5c6c36" is_pre_election: true
10:29:36.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.564900 10421 leader_election.cc:304] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53d1bd87fb0148c780618938c1137c6c; no voters: 39b21963161d48bfa33fa722af5c6c36, a2c6a9ec095f4488875263c4c31e1963
10:29:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.572397 10625 leader_election.cc:290] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.620399 10633 mvcc.cc:204] Tried to move back new op lower bound from 7114493851922186240 to 7114493851488448512. Current Snapshot: MvccSnapshot[applied={T|T < 7114493851922186240}]
10:29:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.638545 10629 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2c6a9ec095f4488875263c4c31e1963. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.639261 10629 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: FOLLOWER
10:29:36.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.639909 10625 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:29:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.642565 10635 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.644567 10635 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: FOLLOWER
10:29:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.646057 10580 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.647147 10580 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: LEADER
10:29:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:36.734514 10651 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:29:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:37.760497 10651 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:37.760789 10651 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: loading cluster ID for follower catalog manager: success
10:29:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:37.797077 10651 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: acquiring CA information for follower catalog manager: success
10:29:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:37.821316 10651 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:29:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:36.450896 10626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:37.827067 10624 debug-util.cc:398] Leaking SignalData structure 0x7b0800006160 after lost signal to thread 10616
10:29:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:38.694474 10616 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.248s user 0.407s sys 0.695s
10:29:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:38.694908 10616 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.248s user 0.408s sys 0.695s
10:29:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:38.697011 10631 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:38.700204 10628 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2254 milliseconds
10:29:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.700237 10616 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:29:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.705484 10616 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.713780 10616 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:38.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.715318 10616 hybrid_clock.cc:648] HybridClock initialized: now 1736936978715241 us; error 78 us; skew 500 ppm
10:29:38.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.716542 10616 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.743077 10616 webserver.cc:458] Webserver started at http://127.10.16.65:44659/ using document root <none> and password file <none>
10:29:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.744346 10616 fs_manager.cc:362] Metadata directory not provided
10:29:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.744624 10616 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.745183 10616 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:29:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.752385 10616 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data/instance:
10:29:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2294858f518148ee8531c8bfb35cd242"
10:29:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:38 on dist-test-slave-gn4w"
10:29:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.753830 10616 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal/instance:
10:29:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2294858f518148ee8531c8bfb35cd242"
10:29:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:38 on dist-test-slave-gn4w"
10:29:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.782275 10616 fs_manager.cc:696] Time spent creating directory manager: real 0.028s user 0.011s sys 0.015s
10:29:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.796936 10659 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.798390 10616 fs_manager.cc:730] Time spent opening block manager: real 0.012s user 0.002s sys 0.007s
10:29:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.798786 10616 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal
10:29:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2294858f518148ee8531c8bfb35cd242"
10:29:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:38 on dist-test-slave-gn4w"
10:29:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.799234 10616 fs_report.cc:389] FS layout report
10:29:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data/data
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:38.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.892268 10616 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:38.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.895442 10616 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.895993 10616 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.922042 10616 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:29:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.922371 10616 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:29:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.922706 10616 ts_tablet_manager.cc:610] Registered 0 tablets
10:29:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:38.922937 10616 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:29:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.264415 10616 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.65:41761
10:29:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.265430 10783 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.65:41761 every 8 connection(s)
10:29:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.267755 10616 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/data/info.pb
10:29:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.277192 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 10616
10:29:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.277757 10305 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-0/wal/instance
10:29:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.293833 10787 heartbeater.cc:346] Connected to a master server at 127.10.16.126:37021
10:29:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.294370 10787 heartbeater.cc:463] Registering TS with master...
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.296939 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/logs
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data/info.pb
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.66:0
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.16.66
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.66
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:29:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.301597 10784 heartbeater.cc:346] Connected to a master server at 127.10.16.124:34587
10:29:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.302162 10784 heartbeater.cc:463] Registering TS with master...
10:29:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.304232 10784 heartbeater.cc:509] Master 127.10.16.124:34587 requested a full tablet report, sending...
10:29:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.313063 10787 heartbeater.cc:509] Master 127.10.16.126:37021 requested a full tablet report, sending...
10:29:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.314950 10524 ts_manager.cc:194] Registered new tserver with Master: 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761)
10:29:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.318686 10350 ts_manager.cc:194] Registered new tserver with Master: 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761)
10:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.319926 10785 heartbeater.cc:346] Connected to a master server at 127.10.16.125:40145
10:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.320389 10785 heartbeater.cc:463] Registering TS with master...
10:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.321588 10785 heartbeater.cc:509] Master 127.10.16.125:40145 requested a full tablet report, sending...
10:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.321624 10350 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.16.65:48909
10:29:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.325935 10435 ts_manager.cc:194] Registered new tserver with Master: 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761)
10:29:39.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.624627 10789 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:39.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.625067 10789 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:39.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.625602 10789 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.656525 10789 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.657047 10789 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:29:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.657217 10789 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:29:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.657359 10789 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:29:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.657745 10789 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.16.66
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.693835 10789 tablet_server_runner.cc:78] Tablet server non-default flags:
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.66:0
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data/info.pb
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.66
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.16.66
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/logs
10:29:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.695431 10789 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:39.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:39.697031 10789 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:39.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.715639 10795 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:40.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:40.327473 10787 heartbeater.cc:501] Master 127.10.16.126:37021 was elected leader, sending a full tablet report...
10:29:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:41.111312 10794 debug-util.cc:398] Leaking SignalData structure 0x7b0800006160 after lost signal to thread 10789
10:29:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:41.647238 10794 kernel_stack_watchdog.cc:198] Thread 10789 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 389ms:
10:29:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:29:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:29:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:29:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:29:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:41.647868 10789 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.928s user 0.678s sys 1.202s
10:29:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:39.720252 10796 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:41.648273 10789 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.929s user 0.678s sys 1.202s
10:29:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:41.650512 10798 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:41.652952 10797 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1921 milliseconds
10:29:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.653023 10789 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:29:41.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.654182 10789 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:41.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.656276 10789 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.658036 10789 hybrid_clock.cc:648] HybridClock initialized: now 1736936981657990 us; error 64 us; skew 500 ppm
10:29:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.658874 10789 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:41.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.665314 10789 webserver.cc:458] Webserver started at http://127.10.16.66:38251/ using document root <none> and password file <none>
10:29:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.666203 10789 fs_manager.cc:362] Metadata directory not provided
10:29:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.666404 10789 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.666800 10789 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:29:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.672077 10789 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data/instance:
10:29:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9128ea62300a43dfb993e431392c642e"
10:29:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:41 on dist-test-slave-gn4w"
10:29:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.673246 10789 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal/instance:
10:29:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9128ea62300a43dfb993e431392c642e"
10:29:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:41 on dist-test-slave-gn4w"
10:29:41.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.680917 10789 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
10:29:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.686197 10808 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.687125 10789 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
10:29:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.687448 10789 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal
10:29:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9128ea62300a43dfb993e431392c642e"
10:29:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:41 on dist-test-slave-gn4w"
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.687772 10789 fs_report.cc:389] FS layout report
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data/data
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.737146 10789 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.738618 10789 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:41.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.739078 10789 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:41.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.748467 10789 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:29:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.748796 10789 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:29:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.749135 10789 ts_tablet_manager.cc:610] Registered 0 tablets
10:29:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.749369 10789 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:29:41.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.924491 10789 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.66:39961
10:29:41.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.925024 10932 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.66:39961 every 8 connection(s)
10:29:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.927208 10789 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/data/info.pb
10:29:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.937275 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 10789
10:29:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.937973 10305 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-1/wal/instance
10:29:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.940138 10935 heartbeater.cc:346] Connected to a master server at 127.10.16.126:37021
10:29:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.940587 10935 heartbeater.cc:463] Registering TS with master...
10:29:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.942421 10935 heartbeater.cc:509] Master 127.10.16.126:37021 requested a full tablet report, sending...
10:29:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.946516 10350 ts_manager.cc:194] Registered new tserver with Master: 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:41.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.947703 10933 heartbeater.cc:346] Connected to a master server at 127.10.16.124:34587
10:29:41.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.948221 10933 heartbeater.cc:463] Registering TS with master...
10:29:41.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.948495 10350 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.16.66:43717
10:29:41.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.949200 10933 heartbeater.cc:509] Master 127.10.16.124:34587 requested a full tablet report, sending...
10:29:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.952456 10524 ts_manager.cc:194] Registered new tserver with Master: 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.955946 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/logs
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data/info.pb
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:41.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.67:0
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.16.67
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.67
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:29:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:29:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.958833 10934 heartbeater.cc:346] Connected to a master server at 127.10.16.125:40145
10:29:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.959162 10934 heartbeater.cc:463] Registering TS with master...
10:29:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.959705 10934 heartbeater.cc:509] Master 127.10.16.125:40145 requested a full tablet report, sending...
10:29:41.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:41.961896 10435 ts_manager.cc:194] Registered new tserver with Master: 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:42.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.301302 10938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:42.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.301856 10938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:42.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.302510 10938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.334841 10938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.335532 10938 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:29:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.335788 10938 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:29:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.336001 10938 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:29:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.336526 10938 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.16.67
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:42.374980 10938 tablet_server_runner.cc:78] Tablet server non-default flags:
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data
10:29:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.67:0
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data/info.pb
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.67
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.16.67
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/logs
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:42.376688 10938 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:42.378276 10938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:42.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.394410 10945 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:42.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:42.952375 10935 heartbeater.cc:501] Master 127.10.16.126:37021 was elected leader, sending a full tablet report...
10:29:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.395294 10946 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:42.397478 10938 server_base.cc:1034] running on GCE node
10:29:43.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:42.395716 10948 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:43.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.604285 10938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:43.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.606487 10938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:43.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.607833 10938 hybrid_clock.cc:648] HybridClock initialized: now 1736936983607798 us; error 44 us; skew 500 ppm
10:29:43.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.608666 10938 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:43.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.615164 10938 webserver.cc:458] Webserver started at http://127.10.16.67:35991/ using document root <none> and password file <none>
10:29:43.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.616075 10938 fs_manager.cc:362] Metadata directory not provided
10:29:43.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.616348 10938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:43.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.616803 10938 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:29:43.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.621624 10938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data/instance:
10:29:43.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e09f5e075d5b46d48a7643f6e5182995"
10:29:43.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:43 on dist-test-slave-gn4w"
10:29:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.622777 10938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal/instance:
10:29:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e09f5e075d5b46d48a7643f6e5182995"
10:29:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:43 on dist-test-slave-gn4w"
10:29:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.630424 10938 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
10:29:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.635986 10955 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.636981 10938 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:29:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.637316 10938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal
10:29:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e09f5e075d5b46d48a7643f6e5182995"
10:29:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:43 on dist-test-slave-gn4w"
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.637665 10938 fs_report.cc:389] FS layout report
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data/data
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:43.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.696161 10938 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:43.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.697809 10938 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:43.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.698354 10938 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:43.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.707562 10938 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:29:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.707834 10938 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:29:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.708163 10938 ts_tablet_manager.cc:610] Registered 0 tablets
10:29:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.708369 10938 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:29:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.920316 10938 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.67:39713
10:29:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.920857 11080 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.67:39713 every 8 connection(s)
10:29:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.922775 10938 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/data/info.pb
10:29:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.930109 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 10938
10:29:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.930583 10305 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/ts-2/wal/instance
10:29:43.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.950309 11081 heartbeater.cc:346] Connected to a master server at 127.10.16.124:34587
10:29:43.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.950850 11081 heartbeater.cc:463] Registering TS with master...
10:29:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.953186 11081 heartbeater.cc:509] Master 127.10.16.124:34587 requested a full tablet report, sending...
10:29:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.953678 11083 heartbeater.cc:346] Connected to a master server at 127.10.16.126:37021
10:29:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.954010 11083 heartbeater.cc:463] Registering TS with master...
10:29:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.955149 11083 heartbeater.cc:509] Master 127.10.16.126:37021 requested a full tablet report, sending...
10:29:43.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.957902 10524 ts_manager.cc:194] Registered new tserver with Master: e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.959990 10350 ts_manager.cc:194] Registered new tserver with Master: e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.961822 10350 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.16.67:45151
10:29:43.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.964196 11082 heartbeater.cc:346] Connected to a master server at 127.10.16.125:40145
10:29:43.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.964553 11082 heartbeater.cc:463] Registering TS with master...
10:29:43.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.965114 11082 heartbeater.cc:509] Master 127.10.16.125:40145 requested a full tablet report, sending...
10:29:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.967537 10435 ts_manager.cc:194] Registered new tserver with Master: e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:43.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:43.974251 10305 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
10:29:43.984 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:29:43.984 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
10:29:43.985 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.16.126"
port: 37021
}
bound_http_address {
host: "127.10.16.126"
port: 32917
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.16.125"
port: 40145
}
bound_http_address {
host: "127.10.16.125"
port: 43507
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.16.124"
port: 34587
}
bound_http_address {
host: "127.10.16.124"
port: 33155
}
}
}
10:29:43.986 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:29:43.987 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.16.65"
port: 41761
}
bound_http_address {
host: "127.10.16.65"
port: 44659
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.16.66"
port: 39961
}
bound_http_address {
host: "127.10.16.66"
port: 38251
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.16.67"
port: 39713
}
bound_http_address {
host: "127.10.16.67"
port: 35991
}
}
}
10:29:43.987 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
10:29:43.988 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
10:29:43.990 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:43.991 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.16.126:37021(127.10.16.126:37021)] connecting to peer
10:29:43.992 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:43.992 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.16.124:34587(127.10.16.124:34587)] Successfully connected to peer
10:29:43.996 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.16.124
10:29:43.998 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.16.126:37021(127.10.16.126:37021)] Successfully connected to peer
10:29:44.001 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.16.126
10:29:44.001 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.16.125:40145(127.10.16.125:40145)] Successfully connected to peer
10:29:44.008 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.16.125
10:29:44.011 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2742eda7, L:/127.0.0.1:51320 - R:/127.10.16.124:34587] using SASL/PLAIN
10:29:44.024 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x91c5f5a9, L:/127.0.0.1:40910 - R:/127.10.16.126:37021] using SASL/PLAIN
10:29:44.024 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145] using SASL/PLAIN
10:29:44.027 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:29:44.027 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=38), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [35ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [37ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [38ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, deferred=null) after lookup
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.029326 10350 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40910:
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:44.032986 10350 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:29:44.066 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table ddb73e40ae9741ba91d634f0be6e9053
10:29:44.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.102021 10867 tablet_service.cc:1467] Processing CreateTablet for tablet 058b80dddb8a4fb1bae9b48f72319b08 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:29:44.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.101936 10868 tablet_service.cc:1467] Processing CreateTablet for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:29:44.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.104497 10867 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 058b80dddb8a4fb1bae9b48f72319b08. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.106585 10719 tablet_service.cc:1467] Processing CreateTablet for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:29:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.108479 10719 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.106575 10718 tablet_service.cc:1467] Processing CreateTablet for tablet 058b80dddb8a4fb1bae9b48f72319b08 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:29:44.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.111320 10868 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.112310 10718 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 058b80dddb8a4fb1bae9b48f72319b08. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.113373 11016 tablet_service.cc:1467] Processing CreateTablet for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:29:44.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.115263 11016 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.120285 11015 tablet_service.cc:1467] Processing CreateTablet for tablet 058b80dddb8a4fb1bae9b48f72319b08 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:29:44.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.121486 11015 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 058b80dddb8a4fb1bae9b48f72319b08. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.146653 11108 tablet_bootstrap.cc:492] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242: Bootstrap starting.
10:29:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.158363 11108 tablet_bootstrap.cc:654] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242: Neither blocks nor log segments found. Creating new log.
10:29:44.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.159246 11107 tablet_bootstrap.cc:492] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e: Bootstrap starting.
10:29:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.160300 11108 log.cc:826] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242: Log is configured to *not* fsync() on all Append() calls
10:29:44.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.163671 11109 tablet_bootstrap.cc:492] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995: Bootstrap starting.
10:29:44.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.166241 11108 tablet_bootstrap.cc:492] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242: No bootstrap required, opened a new log
10:29:44.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.166747 11108 ts_tablet_manager.cc:1397] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242: Time spent bootstrapping tablet: real 0.021s user 0.003s sys 0.009s
10:29:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.166810 11107 tablet_bootstrap.cc:654] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e: Neither blocks nor log segments found. Creating new log.
10:29:44.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.168656 11109 tablet_bootstrap.cc:654] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995: Neither blocks nor log segments found. Creating new log.
10:29:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.168965 11107 log.cc:826] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e: Log is configured to *not* fsync() on all Append() calls
10:29:44.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.170645 11109 log.cc:826] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995: Log is configured to *not* fsync() on all Append() calls
10:29:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.178429 11107 tablet_bootstrap.cc:492] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e: No bootstrap required, opened a new log
10:29:44.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.178851 11107 ts_tablet_manager.cc:1397] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e: Time spent bootstrapping tablet: real 0.021s user 0.014s sys 0.004s
10:29:44.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.178807 11109 tablet_bootstrap.cc:492] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995: No bootstrap required, opened a new log
10:29:44.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.179510 11109 ts_tablet_manager.cc:1397] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995: Time spent bootstrapping tablet: real 0.016s user 0.010s sys 0.005s
10:29:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.199363 11108 raft_consensus.cc:357] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.201385 11108 raft_consensus.cc:383] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.201707 11108 raft_consensus.cc:738] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2294858f518148ee8531c8bfb35cd242, State: Initialized, Role: FOLLOWER
10:29:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.202623 11108 consensus_queue.cc:260] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.203300 11107 raft_consensus.cc:357] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.203998 11107 raft_consensus.cc:383] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.204394 11107 raft_consensus.cc:738] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9128ea62300a43dfb993e431392c642e, State: Initialized, Role: FOLLOWER
10:29:44.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.205420 11107 consensus_queue.cc:260] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.206609 11108 ts_tablet_manager.cc:1428] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242: Time spent starting tablet: real 0.040s user 0.034s sys 0.006s
10:29:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.212376 11109 raft_consensus.cc:357] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.213023 11109 raft_consensus.cc:383] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.213279 11109 raft_consensus.cc:738] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e09f5e075d5b46d48a7643f6e5182995, State: Initialized, Role: FOLLOWER
10:29:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.214040 11109 consensus_queue.cc:260] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.217609 11107 ts_tablet_manager.cc:1428] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e: Time spent starting tablet: real 0.039s user 0.029s sys 0.009s
10:29:44.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.226471 11108 tablet_bootstrap.cc:492] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242: Bootstrap starting.
10:29:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.237104 11109 ts_tablet_manager.cc:1428] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995: Time spent starting tablet: real 0.057s user 0.045s sys 0.007s
10:29:44.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.237993 11083 heartbeater.cc:501] Master 127.10.16.126:37021 was elected leader, sending a full tablet report...
10:29:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.241242 11108 tablet_bootstrap.cc:654] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242: Neither blocks nor log segments found. Creating new log.
10:29:44.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.249636 11109 tablet_bootstrap.cc:492] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995: Bootstrap starting.
10:29:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.254470 11107 tablet_bootstrap.cc:492] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e: Bootstrap starting.
10:29:44.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.259009 11109 tablet_bootstrap.cc:654] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995: Neither blocks nor log segments found. Creating new log.
10:29:44.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.267364 11107 tablet_bootstrap.cc:654] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e: Neither blocks nor log segments found. Creating new log.
10:29:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.270265 11108 tablet_bootstrap.cc:492] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242: No bootstrap required, opened a new log
10:29:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.270742 11108 ts_tablet_manager.cc:1397] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242: Time spent bootstrapping tablet: real 0.045s user 0.011s sys 0.001s
10:29:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.274091 11108 raft_consensus.cc:357] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.275174 11108 raft_consensus.cc:383] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.275580 11108 raft_consensus.cc:738] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2294858f518148ee8531c8bfb35cd242, State: Initialized, Role: FOLLOWER
10:29:44.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.276407 11108 consensus_queue.cc:260] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.279214 11108 ts_tablet_manager.cc:1428] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242: Time spent starting tablet: real 0.008s user 0.003s sys 0.004s
10:29:44.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.282833 11113 raft_consensus.cc:491] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:29:44.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.283382 11113 raft_consensus.cc:513] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.286216 11113 leader_election.cc:290] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961), e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:44.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.298594 11109 tablet_bootstrap.cc:492] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995: No bootstrap required, opened a new log
10:29:44.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.299053 11109 ts_tablet_manager.cc:1397] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995: Time spent bootstrapping tablet: real 0.050s user 0.002s sys 0.035s
10:29:44.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.300837 11107 tablet_bootstrap.cc:492] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e: No bootstrap required, opened a new log
10:29:44.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.301607 11107 ts_tablet_manager.cc:1397] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e: Time spent bootstrapping tablet: real 0.047s user 0.004s sys 0.014s
10:29:44.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.301734 11109 raft_consensus.cc:357] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.302578 11109 raft_consensus.cc:383] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.302896 11109 raft_consensus.cc:738] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e09f5e075d5b46d48a7643f6e5182995, State: Initialized, Role: FOLLOWER
10:29:44.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.303625 11109 consensus_queue.cc:260] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.304842 11107 raft_consensus.cc:357] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.305562 11107 raft_consensus.cc:383] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.305845 11107 raft_consensus.cc:738] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9128ea62300a43dfb993e431392c642e, State: Initialized, Role: FOLLOWER
10:29:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.306541 11107 consensus_queue.cc:260] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.309692 11109 ts_tablet_manager.cc:1428] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995: Time spent starting tablet: real 0.010s user 0.007s sys 0.000s
10:29:44.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:44.313454 10788 tablet.cc:2367] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:29:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.325927 11107 ts_tablet_manager.cc:1428] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e: Time spent starting tablet: real 0.024s user 0.007s sys 0.000s
10:29:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.342944 11036 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "058b80dddb8a4fb1bae9b48f72319b08" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e09f5e075d5b46d48a7643f6e5182995" is_pre_election: true
10:29:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.343848 11036 raft_consensus.cc:2463] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 0.
10:29:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.345144 10662 leader_election.cc:304] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2294858f518148ee8531c8bfb35cd242, e09f5e075d5b46d48a7643f6e5182995; no voters:
10:29:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.345854 11114 raft_consensus.cc:491] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:29:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.346307 10888 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "058b80dddb8a4fb1bae9b48f72319b08" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9128ea62300a43dfb993e431392c642e" is_pre_election: true
10:29:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.346401 11114 raft_consensus.cc:513] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.347261 11113 raft_consensus.cc:2798] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:29:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.347544 11113 raft_consensus.cc:491] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:29:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.347849 10888 raft_consensus.cc:2463] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 0.
10:29:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.348014 11113 raft_consensus.cc:3054] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Advancing to term 1
10:29:44.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.349797 11114 leader_election.cc:290] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761), e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.355002 11113 raft_consensus.cc:513] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.359831 11122 raft_consensus.cc:491] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:29:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.360391 11122 raft_consensus.cc:513] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.361788 10888 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "058b80dddb8a4fb1bae9b48f72319b08" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9128ea62300a43dfb993e431392c642e"
10:29:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.362389 11122 leader_election.cc:290] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961), e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.362547 10888 raft_consensus.cc:3054] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Advancing to term 1
10:29:44.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.366340 11113 leader_election.cc:290] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 election: Requested vote from peers 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961), e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:44.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.366226 10887 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dfd2f9dc6e1840cda5896cbdb73cdc7f" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9128ea62300a43dfb993e431392c642e" is_pre_election: true
10:29:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.366935 10887 raft_consensus.cc:2463] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 0.
10:29:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.367578 10739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "058b80dddb8a4fb1bae9b48f72319b08" candidate_uuid: "9128ea62300a43dfb993e431392c642e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2294858f518148ee8531c8bfb35cd242" is_pre_election: true
10:29:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.368017 10662 leader_election.cc:304] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2294858f518148ee8531c8bfb35cd242, 9128ea62300a43dfb993e431392c642e; no voters:
10:29:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.368585 10739 raft_consensus.cc:2388] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 9128ea62300a43dfb993e431392c642e in current term 1: Already voted for candidate 2294858f518148ee8531c8bfb35cd242 in this term.
10:29:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.370177 11035 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "058b80dddb8a4fb1bae9b48f72319b08" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e09f5e075d5b46d48a7643f6e5182995"
10:29:44.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.370704 11035 raft_consensus.cc:3054] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Advancing to term 1
10:29:44.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.372129 11113 raft_consensus.cc:2798] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:29:44.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.372470 11113 raft_consensus.cc:491] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:29:44.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.372792 11113 raft_consensus.cc:3054] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Advancing to term 1
10:29:44.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.373871 10888 raft_consensus.cc:2463] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 1.
10:29:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.375021 10662 leader_election.cc:304] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2294858f518148ee8531c8bfb35cd242, 9128ea62300a43dfb993e431392c642e; no voters:
10:29:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.375649 11122 raft_consensus.cc:2798] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Leader election won for term 1
10:29:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.376487 11036 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dfd2f9dc6e1840cda5896cbdb73cdc7f" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e09f5e075d5b46d48a7643f6e5182995" is_pre_election: true
10:29:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.377084 11036 raft_consensus.cc:2463] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 0.
10:29:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.379334 11122 raft_consensus.cc:695] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [term 1 LEADER]: Becoming Leader. State: Replica: 2294858f518148ee8531c8bfb35cd242, State: Running, Role: LEADER
10:29:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.378999 11113 raft_consensus.cc:513] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.380187 11122 consensus_queue.cc:237] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.380877 11113 leader_election.cc:290] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 election: Requested vote from peers 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961), e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:44.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.381738 11036 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "058b80dddb8a4fb1bae9b48f72319b08" candidate_uuid: "9128ea62300a43dfb993e431392c642e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e09f5e075d5b46d48a7643f6e5182995" is_pre_election: true
10:29:44.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.382576 11034 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dfd2f9dc6e1840cda5896cbdb73cdc7f" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e09f5e075d5b46d48a7643f6e5182995"
10:29:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.383157 11034 raft_consensus.cc:3054] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Advancing to term 1
10:29:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.383623 10811 leader_election.cc:304] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9128ea62300a43dfb993e431392c642e; no voters: 2294858f518148ee8531c8bfb35cd242, e09f5e075d5b46d48a7643f6e5182995
10:29:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.386361 11035 raft_consensus.cc:2463] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 1.
10:29:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.387367 10888 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dfd2f9dc6e1840cda5896cbdb73cdc7f" candidate_uuid: "2294858f518148ee8531c8bfb35cd242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9128ea62300a43dfb993e431392c642e"
10:29:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.387948 10888 raft_consensus.cc:3054] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Advancing to term 1
10:29:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.388738 11114 raft_consensus.cc:2743] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:29:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.390276 11034 raft_consensus.cc:2463] T dfd2f9dc6e1840cda5896cbdb73cdc7f P e09f5e075d5b46d48a7643f6e5182995 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 1.
10:29:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.391280 10662 leader_election.cc:304] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2294858f518148ee8531c8bfb35cd242, e09f5e075d5b46d48a7643f6e5182995; no voters:
10:29:44.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.392037 11113 raft_consensus.cc:2798] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Leader election won for term 1
10:29:44.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.392489 11113 raft_consensus.cc:695] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [term 1 LEADER]: Becoming Leader. State: Replica: 2294858f518148ee8531c8bfb35cd242, State: Running, Role: LEADER
10:29:44.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.393239 11113 consensus_queue.cc:237] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.396910 10348 catalog_manager.cc:5526] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2294858f518148ee8531c8bfb35cd242 (127.10.16.65). New cstate: current_term: 1 leader_uuid: "2294858f518148ee8531c8bfb35cd242" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } health_report { overall_health: UNKNOWN } } }
10:29:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.397732 10888 raft_consensus.cc:2463] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 9128ea62300a43dfb993e431392c642e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2294858f518148ee8531c8bfb35cd242 in term 1.
10:29:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.417802 10348 catalog_manager.cc:5526] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2294858f518148ee8531c8bfb35cd242 (127.10.16.65). New cstate: current_term: 1 leader_uuid: "2294858f518148ee8531c8bfb35cd242" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } health_report { overall_health: UNKNOWN } } }
10:29:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:44.448191 10936 tablet.cc:2367] T 058b80dddb8a4fb1bae9b48f72319b08 P 9128ea62300a43dfb993e431392c642e: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:29:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:44.456303 11085 tablet.cc:2367] T 058b80dddb8a4fb1bae9b48f72319b08 P e09f5e075d5b46d48a7643f6e5182995: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:29:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.732585 10348 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40922:
10:29:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:29:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:29:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:29:44.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:44.735440 10348 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:29:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.774019 10719 tablet_service.cc:1467] Processing CreateTablet for tablet e97bc8ff74e2499b8a6d0367e55f72cf (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:29:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.774319 11016 tablet_service.cc:1467] Processing CreateTablet for tablet e97bc8ff74e2499b8a6d0367e55f72cf (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:29:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.775743 10719 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e97bc8ff74e2499b8a6d0367e55f72cf. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.775736 11016 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e97bc8ff74e2499b8a6d0367e55f72cf. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.781893 10868 tablet_service.cc:1467] Processing CreateTablet for tablet e97bc8ff74e2499b8a6d0367e55f72cf (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:29:44.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.783510 10868 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e97bc8ff74e2499b8a6d0367e55f72cf. 1 dirs total, 0 dirs full, 0 dirs failed
10:29:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.790714 11109 tablet_bootstrap.cc:492] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: Bootstrap starting.
10:29:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.796638 11138 tablet_bootstrap.cc:492] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: Bootstrap starting.
10:29:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.797142 11109 tablet_bootstrap.cc:654] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: Neither blocks nor log segments found. Creating new log.
10:29:44.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.798076 11107 tablet_bootstrap.cc:492] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: Bootstrap starting.
10:29:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.802456 11109 tablet_bootstrap.cc:492] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: No bootstrap required, opened a new log
10:29:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.802784 11109 ts_tablet_manager.cc:1397] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.001s
10:29:44.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.803143 11107 tablet_bootstrap.cc:654] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: Neither blocks nor log segments found. Creating new log.
10:29:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.804944 11109 raft_consensus.cc:357] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.806667 11122 consensus_queue.cc:1035] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.806048 11109 raft_consensus.cc:383] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.807587 11109 raft_consensus.cc:738] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e09f5e075d5b46d48a7643f6e5182995, State: Initialized, Role: FOLLOWER
10:29:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.808255 11109 consensus_queue.cc:260] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.811419 11141 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:44.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.812366 11141 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:44.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.814735 11138 tablet_bootstrap.cc:654] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: Neither blocks nor log segments found. Creating new log.
10:29:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.819113 11107 tablet_bootstrap.cc:492] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: No bootstrap required, opened a new log
10:29:44.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.819964 11107 ts_tablet_manager.cc:1397] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: Time spent bootstrapping tablet: real 0.022s user 0.008s sys 0.002s
10:29:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.822737 11107 raft_consensus.cc:357] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.824133 11109 ts_tablet_manager.cc:1428] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: Time spent starting tablet: real 0.021s user 0.008s sys 0.000s
10:29:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.823923 11107 raft_consensus.cc:383] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.824505 11107 raft_consensus.cc:738] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9128ea62300a43dfb993e431392c642e, State: Initialized, Role: FOLLOWER
10:29:44.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.825233 11107 consensus_queue.cc:260] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.826653 11138 tablet_bootstrap.cc:492] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: No bootstrap required, opened a new log
10:29:44.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.827723 11107 ts_tablet_manager.cc:1428] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: Time spent starting tablet: real 0.007s user 0.005s sys 0.000s
10:29:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.827064 11138 ts_tablet_manager.cc:1397] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: Time spent bootstrapping tablet: real 0.031s user 0.013s sys 0.003s
10:29:44.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.830739 11138 raft_consensus.cc:357] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.831465 11138 raft_consensus.cc:383] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:29:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.831746 11138 raft_consensus.cc:738] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2294858f518148ee8531c8bfb35cd242, State: Initialized, Role: FOLLOWER
10:29:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.833681 11114 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.834388 11114 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:44.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.832458 11138 consensus_queue.cc:260] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.838945 11124 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.839787 11124 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:44.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.844009 11138 ts_tablet_manager.cc:1428] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: Time spent starting tablet: real 0.016s user 0.006s sys 0.000s
10:29:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.858034 11124 consensus_queue.cc:1035] T dfd2f9dc6e1840cda5896cbdb73cdc7f P 2294858f518148ee8531c8bfb35cd242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:44.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.923840 11113 consensus_queue.cc:1035] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:44.939396 11122 consensus_queue.cc:1035] T 058b80dddb8a4fb1bae9b48f72319b08 P 2294858f518148ee8531c8bfb35cd242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.078781 11142 raft_consensus.cc:491] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:29:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.079185 11142 raft_consensus.cc:513] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.081457 11142 leader_election.cc:290] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761), 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:45.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.091500 10739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e97bc8ff74e2499b8a6d0367e55f72cf" candidate_uuid: "e09f5e075d5b46d48a7643f6e5182995" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2294858f518148ee8531c8bfb35cd242" is_pre_election: true
10:29:45.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.091998 10739 raft_consensus.cc:2463] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e09f5e075d5b46d48a7643f6e5182995 in term 0.
10:29:45.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.092999 10958 leader_election.cc:304] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2294858f518148ee8531c8bfb35cd242, e09f5e075d5b46d48a7643f6e5182995; no voters:
10:29:45.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.093300 10888 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e97bc8ff74e2499b8a6d0367e55f72cf" candidate_uuid: "e09f5e075d5b46d48a7643f6e5182995" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9128ea62300a43dfb993e431392c642e" is_pre_election: true
10:29:45.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.093719 11142 raft_consensus.cc:2798] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:29:45.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.093968 11142 raft_consensus.cc:491] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:29:45.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.093855 10888 raft_consensus.cc:2463] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e09f5e075d5b46d48a7643f6e5182995 in term 0.
10:29:45.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.094166 11142 raft_consensus.cc:3054] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 0 FOLLOWER]: Advancing to term 1
10:29:45.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.098429 11142 raft_consensus.cc:513] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:45.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.099853 11142 leader_election.cc:290] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [CANDIDATE]: Term 1 election: Requested vote from peers 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761), 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:45.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.100543 10739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e97bc8ff74e2499b8a6d0367e55f72cf" candidate_uuid: "e09f5e075d5b46d48a7643f6e5182995" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2294858f518148ee8531c8bfb35cd242"
10:29:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.100760 10888 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e97bc8ff74e2499b8a6d0367e55f72cf" candidate_uuid: "e09f5e075d5b46d48a7643f6e5182995" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9128ea62300a43dfb993e431392c642e"
10:29:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.101015 10739 raft_consensus.cc:3054] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 0 FOLLOWER]: Advancing to term 1
10:29:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.101248 10888 raft_consensus.cc:3054] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 0 FOLLOWER]: Advancing to term 1
10:29:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.105015 10739 raft_consensus.cc:2463] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e09f5e075d5b46d48a7643f6e5182995 in term 1.
10:29:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.105050 10888 raft_consensus.cc:2463] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e09f5e075d5b46d48a7643f6e5182995 in term 1.
10:29:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.105823 10958 leader_election.cc:304] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2294858f518148ee8531c8bfb35cd242, e09f5e075d5b46d48a7643f6e5182995; no voters:
10:29:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.106393 11142 raft_consensus.cc:2798] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 1 FOLLOWER]: Leader election won for term 1
10:29:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.106803 11142 raft_consensus.cc:695] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [term 1 LEADER]: Becoming Leader. State: Replica: e09f5e075d5b46d48a7643f6e5182995, State: Running, Role: LEADER
10:29:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.107741 11142 consensus_queue.cc:237] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } }
10:29:45.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.114383 11141 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: TxnStatusTablet state changed. Reason: New leader e09f5e075d5b46d48a7643f6e5182995. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.114967 11141 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: This TxnStatusTablet replica's current role is: LEADER
10:29:45.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.117324 11156 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
10:29:45.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.117843 11156 txn_status_manager.cc:930] Loading transaction status metadata into memory...
10:29:45.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.119302 10348 catalog_manager.cc:5526] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 reported cstate change: term changed from 0 to 1, leader changed from <none> to e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67). New cstate: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } health_report { overall_health: HEALTHY } } }
10:29:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.235288 10888 raft_consensus.cc:1270] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e [term 1 FOLLOWER]: Refusing update from remote peer e09f5e075d5b46d48a7643f6e5182995: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:29:45.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.236462 11141 consensus_queue.cc:1035] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:45.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.236691 10739 raft_consensus.cc:1270] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242 [term 1 FOLLOWER]: Refusing update from remote peer e09f5e075d5b46d48a7643f6e5182995: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:29:45.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.241887 11141 consensus_queue.cc:1035] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:29:45.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.264605 11114 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: TxnStatusTablet state changed. Reason: New leader e09f5e075d5b46d48a7643f6e5182995. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.266518 11146 mvcc.cc:204] Tried to move back new op lower bound from 7114493891507945472 to 7114493891007963136. Current Snapshot: MvccSnapshot[applied={T|T < 7114493891507945472}]
10:29:45.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.283838 11141 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.284642 11141 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: This TxnStatusTablet replica's current role is: LEADER
10:29:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.284848 11114 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:45.298 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
10:29:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.301440 11141 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.302215 11141 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995: This TxnStatusTablet replica's current role is: LEADER
10:29:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.303124 11113 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: TxnStatusTablet state changed. Reason: New leader e09f5e075d5b46d48a7643f6e5182995. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.303836 11113 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:45.304 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table ddb73e40ae9741ba91d634f0be6e9053
10:29:45.304 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.16.126:37021
10:29:45.308 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:29:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.308732 11145 mvcc.cc:204] Tried to move back new op lower bound from 7114493891507945472 to 7114493891007963136. Current Snapshot: MvccSnapshot[applied={T|T < 7114493891507945472}]
10:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.309247 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 10317
10:29:45.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.334091 11124 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.334786 11124 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P 2294858f518148ee8531c8bfb35cd242: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.339085 11114 tablet_replica.cc:440] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e09f5e075d5b46d48a7643f6e5182995" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2294858f518148ee8531c8bfb35cd242" member_type: VOTER last_known_addr { host: "127.10.16.65" port: 41761 } } peers { permanent_uuid: "9128ea62300a43dfb993e431392c642e" member_type: VOTER last_known_addr { host: "127.10.16.66" port: 39961 } } peers { permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" member_type: VOTER last_known_addr { host: "127.10.16.67" port: 39713 } } }
10:29:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.339761 11114 tablet_replica.cc:443] T e97bc8ff74e2499b8a6d0367e55f72cf P 9128ea62300a43dfb993e431392c642e: This TxnStatusTablet replica's current role is: FOLLOWER
10:29:45.383 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.16.126:37021(127.10.16.126:37021)] handling channelInactive
10:29:45.384 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.126:37021(127.10.16.126:37021)] cleaning up while in state READY due to: connection closed
10:29:45.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.384819 10811 connection.cc:537] client connection to 127.10.16.126:37021 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:29:45.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.385854 11083 heartbeater.cc:646] Failed to heartbeat to 127.10.16.126:37021 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.10.16.126:37021 (error 108)
10:29:45.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:29:45.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.386806 10935 heartbeater.cc:646] Failed to heartbeat to 127.10.16.126:37021 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:29:45.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.387558 10662 connection.cc:537] client connection to 127.10.16.126:37021 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:29:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.388080 10787 heartbeater.cc:646] Failed to heartbeat to 127.10.16.126:37021 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:29:45.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.534044 11166 raft_consensus.cc:491] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a2c6a9ec095f4488875263c4c31e1963)
10:29:45.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.534471 11166 raft_consensus.cc:513] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.535902 11166 leader_election.cc:290] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.538168 10421 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111) [suppressed 6 similar messages]
10:29:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.538337 10534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "53d1bd87fb0148c780618938c1137c6c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "39b21963161d48bfa33fa722af5c6c36" is_pre_election: true
10:29:45.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.541263 10421 leader_election.cc:336] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111)
10:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.541676 10421 leader_election.cc:304] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53d1bd87fb0148c780618938c1137c6c; no voters: 39b21963161d48bfa33fa722af5c6c36, a2c6a9ec095f4488875263c4c31e1963
10:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.542362 11166 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
10:29:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.551771 11169 raft_consensus.cc:491] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a2c6a9ec095f4488875263c4c31e1963)
10:29:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.552222 11169 raft_consensus.cc:513] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.554534 11169 leader_election.cc:290] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021), 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145)
10:29:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.555225 10441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "39b21963161d48bfa33fa722af5c6c36" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "53d1bd87fb0148c780618938c1137c6c" is_pre_election: true
10:29:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.555955 10441 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 39b21963161d48bfa33fa722af5c6c36 in term 1.
10:29:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.557400 10508 leader_election.cc:304] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39b21963161d48bfa33fa722af5c6c36, 53d1bd87fb0148c780618938c1137c6c; no voters:
10:29:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.558068 10510 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111)
10:29:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.558239 11169 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 1 FOLLOWER]: Leader pre-election won for term 2
10:29:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.558535 11169 raft_consensus.cc:491] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 1 FOLLOWER]: Starting leader election (detected failure of leader a2c6a9ec095f4488875263c4c31e1963)
10:29:45.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.558900 11169 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 1 FOLLOWER]: Advancing to term 2
10:29:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.562952 10510 leader_election.cc:336] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111)
10:29:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.565802 11169 raft_consensus.cc:513] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.567231 11169 leader_election.cc:290] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [CANDIDATE]: Term 2 election: Requested vote from peers a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021), 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145)
10:29:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.568303 10441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "39b21963161d48bfa33fa722af5c6c36" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "53d1bd87fb0148c780618938c1137c6c"
10:29:45.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.568850 10441 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 1 FOLLOWER]: Advancing to term 2
10:29:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.570842 10510 leader_election.cc:336] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111)
10:29:45.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.575820 10441 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 39b21963161d48bfa33fa722af5c6c36 in term 2.
10:29:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.576678 10508 leader_election.cc:304] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 39b21963161d48bfa33fa722af5c6c36, 53d1bd87fb0148c780618938c1137c6c; no voters: a2c6a9ec095f4488875263c4c31e1963
10:29:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.577266 11169 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 FOLLOWER]: Leader election won for term 2
10:29:45.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.578841 11169 raft_consensus.cc:695] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 LEADER]: Becoming Leader. State: Replica: 39b21963161d48bfa33fa722af5c6c36, State: Running, Role: LEADER
10:29:45.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.579564 11169 consensus_queue.cc:237] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.582522 11172 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 39b21963161d48bfa33fa722af5c6c36. Latest consensus state: current_term: 2 leader_uuid: "39b21963161d48bfa33fa722af5c6c36" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.583092 11172 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: LEADER
10:29:45.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.585605 11173 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:29:45.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.595357 11173 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]
10:29:45.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.597079 11173 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98]
10:29:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.604038 11173 tablet_loader.cc:96] loaded metadata for tablet 058b80dddb8a4fb1bae9b48f72319b08 (table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053])
10:29:45.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.605576 11173 tablet_loader.cc:96] loaded metadata for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f (table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053])
10:29:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.606877 11173 tablet_loader.cc:96] loaded metadata for tablet e97bc8ff74e2499b8a6d0367e55f72cf (table kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98])
10:29:45.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.608413 11173 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:29:45.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.613008 11173 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:45.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.613358 11173 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:29:45.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.619065 11173 catalog_manager.cc:1505] Loading token signing keys...
10:29:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.623581 11173 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Loaded TSK: 0
10:29:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.624996 11173 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:29:45.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.674484 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:29:45.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.690659 10441 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Refusing update from remote peer 39b21963161d48bfa33fa722af5c6c36: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
10:29:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.691921 11172 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
10:29:45.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.695174 11166 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 39b21963161d48bfa33fa722af5c6c36. Latest consensus state: current_term: 2 leader_uuid: "39b21963161d48bfa33fa722af5c6c36" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:45.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.695977 11166 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: FOLLOWER
10:29:45.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.700814 11166 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "39b21963161d48bfa33fa722af5c6c36" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:45.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.701639 11166 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: FOLLOWER
10:29:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.700632 11172 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "39b21963161d48bfa33fa722af5c6c36" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.702411 11172 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: LEADER
10:29:45.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.126:37021(127.10.16.126:37021)] connecting to peer
10:29:45.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021
10:29:45.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.126:37021(127.10.16.126:37021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021
10:29:45.811 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.10.16.126:37021(127.10.16.126:37021) for tablet Kudu Master: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021
10:29:45.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.126:37021(127.10.16.126:37021)] connecting to peer
10:29:45.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021
10:29:45.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.126:37021(127.10.16.126:37021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021
10:29:45.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.126:37021: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021
10:29:45.835 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:29:45.835 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=26), Traces: [0ms] sending RPC to server master-127.10.16.126:37021, [2ms] received response from server master-127.10.16.126:37021: Network error: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: Network error: Failed to connect to peer master-127.10.16.126:37021(127.10.16.126:37021): Connection refused: /127.10.16.126:37021, [25ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, deferred=Deferred@1650999553(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/333194585@719a5aa8, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1704246467@2a47800e)) after lookup
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:45.902185 10523 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51346:
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:29:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:45.905450 10523 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:29:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:46.180930 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:29:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:46.666721 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:29:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.158099 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:29:47.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.329095 11081 heartbeater.cc:501] Master 127.10.16.124:34587 was elected leader, sending a full tablet report...
10:29:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.352388 10933 heartbeater.cc:501] Master 127.10.16.124:34587 was elected leader, sending a full tablet report...
10:29:47.394 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet dfd2f9dc6e1840cda5896cbdb73cdc7f for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
10:29:47.394 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 058b80dddb8a4fb1bae9b48f72319b08 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
10:29:47.395 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=dfd2f9dc6e1840cda5896cbdb73cdc7f@[2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)[L],9128ea62300a43dfb993e431392c642e(127.10.16.66:39961),e09f5e075d5b46d48a7643f6e5182995(127.10.16.67:39713)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=058b80dddb8a4fb1bae9b48f72319b08@[2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)[L],9128ea62300a43dfb993e431392c642e(127.10.16.66:39961),e09f5e075d5b46d48a7643f6e5182995(127.10.16.67:39713)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
10:29:47.395 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.16.124:34587, [4ms] Sub RPC GetTableLocations: received response from server master-127.10.16.124:34587: OK, deferred=null) after lookup
10:29:47.395 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)] connecting to peer
10:29:47.396 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)] Successfully connected to peer
10:29:47.411 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xba994e06, L:/127.0.0.1:48010 - R:/127.10.16.65:41761] using TOKEN/null
10:29:47.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.417148 10676 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.10.16.126:37021)
10:29:47.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.424959 10784 heartbeater.cc:501] Master 127.10.16.124:34587 was elected leader, sending a full tablet report...
10:29:47.488 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.16.126:37021
10:29:47.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:29:47.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.490305 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:47.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/logs
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/info.pb
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.126:37021
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.126
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32917
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:29:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:29:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.710141 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:29:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.809362 11217 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.809903 11217 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.810371 11217 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.844316 11217 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:29:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.844596 11217 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:29:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.844761 11217 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.844954 11217 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:29:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.845144 11217 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:29:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.845328 11217 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:29:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.845479 11217 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36871/jwks.json
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.881919 11217 master_runner.cc:386] Master server non-default flags:
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.126:37021
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/info.pb
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.126
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32917
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/logs
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.883247 11217 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:47.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:47.884789 11217 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.900180 11224 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:48.222121 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
10:29:48.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:48.743811 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
10:29:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.902448 11226 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:47.901237 11223 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.148626 11225 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:29:49.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.149168 11217 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:29:49.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.153143 11217 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:49.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.156133 11217 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:49.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.157567 11217 hybrid_clock.cc:648] HybridClock initialized: now 1736936989157509 us; error 64 us; skew 500 ppm
10:29:49.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.158337 11217 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:49.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.170004 11217 webserver.cc:458] Webserver started at http://127.10.16.126:32917/ using document root <none> and password file <none>
10:29:49.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.171155 11217 fs_manager.cc:362] Metadata directory not provided
10:29:49.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.171459 11217 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.180253 11217 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
10:29:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.184593 11237 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.185609 11217 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.185881 11217 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2c6a9ec095f4488875263c4c31e1963"
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:31 on dist-test-slave-gn4w"
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.188215 11217 fs_report.cc:389] FS layout report
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/wal
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/data
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:49.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.251910 11217 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.253832 11217 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.254370 11217 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:49.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.279942 10510 consensus_peers.cc:487] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 -> Peer a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021): Couldn't send request to peer a2c6a9ec095f4488875263c4c31e1963. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.126:37021: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
10:29:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.351162 11217 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.126:37021
10:29:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.351287 11299 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.126:37021 every 8 connection(s)
10:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.354038 11217 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-0/data/info.pb
10:29:49.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.363718 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 11217
10:29:49.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:29:49.364 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.16.124:34587
10:29:49.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:29:49.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.364890 11300 sys_catalog.cc:263] Verifying existing consensus state
10:29:49.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.365020 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 10482
10:29:49.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.370752 11300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Bootstrap starting.
10:29:49.395 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.16.124:34587(127.10.16.124:34587)] handling channelInactive
10:29:49.395 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state READY due to: connection closed
10:29:49.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:29:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.414558 11300 log.cc:826] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Log is configured to *not* fsync() on all Append() calls
10:29:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.422216 10935 heartbeater.cc:346] Connected to a master server at 127.10.16.126:37021
10:29:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.444471 10787 heartbeater.cc:346] Connected to a master server at 127.10.16.126:37021
10:29:49.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.454352 11300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:29:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.455160 11300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Bootstrap complete.
10:29:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.476457 11300 raft_consensus.cc:357] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:49.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.478811 11300 raft_consensus.cc:738] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2c6a9ec095f4488875263c4c31e1963, State: Initialized, Role: FOLLOWER
10:29:49.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.479596 11300 consensus_queue.cc:260] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:49.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.481988 11307 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.482620 11307 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: FOLLOWER
10:29:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.483450 11300 sys_catalog.cc:564] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: configured and running, proceeding with master startup.
10:29:49.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.513922 11318 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:49.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.514258 11318 catalog_manager.cc:1553] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: loading cluster ID for follower catalog manager: success
10:29:49.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.521512 11318 catalog_manager.cc:1575] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: acquiring CA information for follower catalog manager: success
10:29:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.528048 11318 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:29:49.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.548194 11217 master_runner.cc:186] Error getting master registration for 127.10.16.124:34587: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111),
10:29:49.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.567164 11217 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:29:49.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:49.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.571 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.10.16.124:34587(127.10.16.124:34587) for tablet Kudu Master: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:49.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.126:37021(127.10.16.126:37021)] connecting to peer
10:29:49.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.591 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.16.126:37021(127.10.16.126:37021)] Successfully connected to peer
10:29:49.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.609 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5febf0ee, L:/127.0.0.1:51860 - R:/127.10.16.126:37021] using TOKEN/null
10:29:49.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:49.629 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:49.630 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.631 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:49.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.644734 11324 raft_consensus.cc:491] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 39b21963161d48bfa33fa722af5c6c36)
10:29:49.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.645113 11324 raft_consensus.cc:513] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:49.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.646657 11324 leader_election.cc:290] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:49.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:49.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.663297 10421 leader_election.cc:336] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.663268 11264 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "53d1bd87fb0148c780618938c1137c6c" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "a2c6a9ec095f4488875263c4c31e1963" is_pre_election: true
10:29:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.664283 11264 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53d1bd87fb0148c780618938c1137c6c in term 1.
10:29:49.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.665333 10421 leader_election.cc:304] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 53d1bd87fb0148c780618938c1137c6c, a2c6a9ec095f4488875263c4c31e1963; no voters: 39b21963161d48bfa33fa722af5c6c36
10:29:49.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.666064 11324 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Leader pre-election won for term 3
10:29:49.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.666288 11324 raft_consensus.cc:491] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Starting leader election (detected failure of leader 39b21963161d48bfa33fa722af5c6c36)
10:29:49.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.666510 11324 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 2 FOLLOWER]: Advancing to term 3
10:29:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.671023 11324 raft_consensus.cc:513] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.672519 11324 leader_election.cc:290] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 3 election: Requested vote from peers a2c6a9ec095f4488875263c4c31e1963 (127.10.16.126:37021), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.673195 11264 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "53d1bd87fb0148c780618938c1137c6c" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "a2c6a9ec095f4488875263c4c31e1963"
10:29:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.673761 11264 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 1 FOLLOWER]: Advancing to term 3
10:29:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.676357 10421 leader_election.cc:336] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111)
10:29:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.680886 11264 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53d1bd87fb0148c780618938c1137c6c in term 3.
10:29:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.681671 10421 leader_election.cc:304] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 53d1bd87fb0148c780618938c1137c6c, a2c6a9ec095f4488875263c4c31e1963; no voters: 39b21963161d48bfa33fa722af5c6c36
10:29:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.682371 11324 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 3 FOLLOWER]: Leader election won for term 3
10:29:49.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.683650 11324 raft_consensus.cc:695] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [term 3 LEADER]: Becoming Leader. State: Replica: 53d1bd87fb0148c780618938c1137c6c, State: Running, Role: LEADER
10:29:49.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.684477 11324 consensus_queue.cc:237] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.688271 11328 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 53d1bd87fb0148c780618938c1137c6c. Latest consensus state: current_term: 3 leader_uuid: "53d1bd87fb0148c780618938c1137c6c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.688997 11328 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: LEADER
10:29:49.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.690935 11330 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:29:49.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.702273 11330 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]
10:29:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.703716 11330 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98]
10:29:49.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.708612 11330 tablet_loader.cc:96] loaded metadata for tablet 058b80dddb8a4fb1bae9b48f72319b08 (table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053])
10:29:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.709908 11330 tablet_loader.cc:96] loaded metadata for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f (table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053])
10:29:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.711263 11330 tablet_loader.cc:96] loaded metadata for tablet e97bc8ff74e2499b8a6d0367e55f72cf (table kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98])
10:29:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.712683 11330 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:29:49.714 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.716428 11330 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.716609 11330 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:29:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.719817 11330 catalog_manager.cc:1505] Loading token signing keys...
10:29:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.723227 11330 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c: Loaded TSK: 0
10:29:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.724229 11330 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:29:49.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:49.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587
10:29:49.754 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:29:49.755 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=184), Traces: [0ms] sending RPC to server master-127.10.16.124:34587, [1ms] received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [27ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [84ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, deferred=Deferred@525075174(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/333194585@36e6137c -> (continuation of Deferred@146915409 after retry RPC after error@1707315674) -> (continuation of Deferred@451404982 after retry RPC after error@471628133) -> (continuation of Deferred@450490879 after retry RPC after error@1668152895) -> (continuation of Deferred@1175090352 after retry RPC after error@958593605), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1704246467@6ca1b297 -> (continuation of Deferred@146915409 after retry RPC after error@1707315674) -> (continuation of Deferred@451404982 after retry RPC after error@471628133) -> (continuation of Deferred@450490879 after retry RPC after error@1668152895) -> (continuation of Deferred@1175090352 after retry RPC after error@958593605))) after lookup
10:29:49.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.768242 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:29:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.794399 11264 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Refusing update from remote peer 53d1bd87fb0148c780618938c1137c6c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
10:29:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.795691 11328 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
10:29:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.805948 11341 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "53d1bd87fb0148c780618938c1137c6c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.807093 11341 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: FOLLOWER
10:29:49.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.811962 11307 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 53d1bd87fb0148c780618938c1137c6c. Latest consensus state: current_term: 3 leader_uuid: "53d1bd87fb0148c780618938c1137c6c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:49.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.812808 11307 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: FOLLOWER
10:29:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.814560 11324 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "53d1bd87fb0148c780618938c1137c6c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.815126 11324 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c [sys.catalog]: This master's current role is: LEADER
10:29:49.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.817072 11307 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "53d1bd87fb0148c780618938c1137c6c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:49.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.817785 11307 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: FOLLOWER
10:29:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.821858 10465 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:43058 (request call id 7) took 65 ms (client timeout 65 ms). Trace:
10:29:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.822278 10465 rpcz_store.cc:269] 0115 10:29:49.756407 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:29:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:49.756587 (+ 180us) service_pool.cc:225] Handling call
10:29:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:49.821637 (+ 65050us) inbound_call.cc:173] Queueing success response
10:29:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":776,"init.queue_time_us":2012,"thread_start_us":8601,"threads_started":5}
10:29:49.824 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=253), Traces: [0ms] sending RPC to server master-127.10.16.124:34587, [1ms] received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [27ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [84ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [185ms] sending RPC to server master-127.10.16.125:40145, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@525075174(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/333194585@36e6137c -> (continuation of Deferred@146915409 after retry RPC after error@1707315674) -> (continuation of Deferred@451404982 after retry RPC after error@471628133) -> (continuation of Deferred@450490879 after retry RPC after error@1668152895) -> (continuation of Deferred@1175090352 after retry RPC after error@958593605) -> (continuation of Deferred@1622222138 after retry RPC@1963680360), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1704246467@6ca1b297 -> (continuation of Deferred@146915409 after retry RPC after error@1707315674) -> (continuation of Deferred@451404982 after retry RPC after error@471628133) -> (continuation of Deferred@450490879 after retry RPC after error@1668152895) -> (continuation of Deferred@1175090352 after retry RPC after error@958593605) -> (continuation of Deferred@1622222138 after retry RPC@1963680360)))
10:29:49.825 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=253), Traces: [0ms] sending RPC to server master-127.10.16.124:34587, [1ms] received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [27ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [84ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Network error: Failed to connect to peer master-127.10.16.124:34587(127.10.16.124:34587): Connection refused: /127.10.16.124:34587, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: OK, [185ms] sending RPC to server master-127.10.16.125:40145, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@525075174(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/333194585@36e6137c -> (continuation of Deferred@146915409 after retry RPC after error@1707315674) -> (continuation of Deferred@451404982 after retry RPC after error@471628133) -> (continuation of Deferred@450490879 after retry RPC after error@1668152895) -> (continuation of Deferred@1175090352 after retry RPC after error@958593605) -> (continuation of Deferred@1622222138 after retry RPC@1963680360), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1704246467@6ca1b297 -> (continuation of Deferred@146915409 after retry RPC after error@1707315674) -> (continuation of Deferred@451404982 after retry RPC after error@471628133) -> (continuation of Deferred@450490879 after retry RPC after error@1668152895) -> (continuation of Deferred@1175090352 after retry RPC after error@958593605) -> (continuation of Deferred@1622222138 after retry RPC@1963680360)))
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:49.794231 10435 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47278:
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:29:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:49.832309 10435 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:29:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:50.326598 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:29:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.345366 11083 heartbeater.cc:346] Connected to a master server at 127.10.16.126:37021
10:29:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.352427 11254 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" instance_seqno: 1736936983889208) as {username='slave'} at 127.10.16.67:45595; Asking this server to re-register.
10:29:50.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.353802 11083 heartbeater.cc:463] Registering TS with master...
10:29:50.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.357457 11254 ts_manager.cc:194] Registered new tserver with Master: e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.429713 11254 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9128ea62300a43dfb993e431392c642e" instance_seqno: 1736936981894189) as {username='slave'} at 127.10.16.66:54287; Asking this server to re-register.
10:29:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.431211 10935 heartbeater.cc:463] Registering TS with master...
10:29:50.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.434630 11254 ts_manager.cc:194] Registered new tserver with Master: 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:50.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.457441 11254 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "2294858f518148ee8531c8bfb35cd242" instance_seqno: 1736936979208743) as {username='slave'} at 127.10.16.65:38503; Asking this server to re-register.
10:29:50.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.458685 10787 heartbeater.cc:463] Registering TS with master...
10:29:50.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:50.461606 11254 ts_manager.cc:194] Registered new tserver with Master: 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761)
10:29:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:50.553984 10421 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111) [suppressed 9 similar messages]
10:29:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:50.881840 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:29:51.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.344134 11082 heartbeater.cc:501] Master 127.10.16.125:40145 was elected leader, sending a full tablet report...
10:29:51.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.367715 10934 heartbeater.cc:501] Master 127.10.16.125:40145 was elected leader, sending a full tablet report...
10:29:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.438280 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:29:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.440068 10785 heartbeater.cc:501] Master 127.10.16.125:40145 was elected leader, sending a full tablet report...
10:29:51.452 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.16.124:34587
10:29:51.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.455021 10305 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/logs
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/info.pb
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.124:34587
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.124
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33155
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:29:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:29:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.774420 11370 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:29:51.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.774927 11370 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:29:51.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.775434 11370 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:29:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.805963 11370 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:29:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.806231 11370 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:29:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.806403 11370 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:29:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.806596 11370 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:29:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.806785 11370 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:29:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.806970 11370 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:29:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.807111 11370 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36871/jwks.json
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.844821 11370 master_runner.cc:386] Master server non-default flags:
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.16.84:44841
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.16.126:37021,127.10.16.125:40145,127.10.16.124:34587
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.16.124:34587
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36871/jwks.json
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/info.pb
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.16.124
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33155
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/logs
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:29:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:29:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:29:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:29:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:29:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:29:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:29:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.846060 11370 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:29:51.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:51.847568 11370 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:29:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.862246 11376 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.920607 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:29:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:52.447381 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
10:29:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:52.942117 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
10:29:53.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:51.863575 11377 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:53.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.223498 11378 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:29:53.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.226223 11379 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:29:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.228750 11370 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.370s user 0.557s sys 0.803s
10:29:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.229195 11370 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.371s user 0.557s sys 0.804s
10:29:53.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.229493 11370 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:29:53.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.231122 11370 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:29:53.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.234412 11370 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:29:53.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.235872 11370 hybrid_clock.cc:648] HybridClock initialized: now 1736936993235819 us; error 37 us; skew 500 ppm
10:29:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.237147 11370 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:29:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.246845 11370 webserver.cc:458] Webserver started at http://127.10.16.124:33155/ using document root <none> and password file <none>
10:29:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.248389 11370 fs_manager.cc:362] Metadata directory not provided
10:29:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.248723 11370 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:29:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.262622 11370 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.006s sys 0.002s
10:29:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.268937 11394 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:29:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.270233 11370 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
10:29:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.270695 11370 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data,/tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b21963161d48bfa33fa722af5c6c36"
10:29:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:29:35 on dist-test-slave-gn4w"
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.274647 11370 fs_report.cc:389] FS layout report
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/wal
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/data
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:29:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:29:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.373930 11370 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:29:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.375731 11370 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:29:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.376293 11370 kserver.cc:163] Server-wide thread pool size limit: 3276
10:29:53.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.441303 10421 consensus_peers.cc:487] T 00000000000000000000000000000000 P 53d1bd87fb0148c780618938c1137c6c -> Peer 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587): Couldn't send request to peer 39b21963161d48bfa33fa722af5c6c36. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.124:34587: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
10:29:53.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.465607 11370 rpc_server.cc:307] RPC server started. Bound to: 127.10.16.124:34587
10:29:53.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.465795 11457 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.16.124:34587 every 8 connection(s)
10:29:53.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.469431 11370 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskM5Ix5b/test-tmp/mini-kudu-cluster7363512361576707371/master-2/data/info.pb
10:29:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.478830 10305 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu as pid 11370
10:29:53.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:29:53.479 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.16.125:40145
10:29:53.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:29:53.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.481068 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 10397
10:29:53.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.485414 11460 sys_catalog.cc:263] Verifying existing consensus state
10:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.492501 10784 heartbeater.cc:346] Connected to a master server at 127.10.16.124:34587
10:29:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.494658 11460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Bootstrap starting.
10:29:53.525 [ERROR - kudu-nio-0] (Connection.java:452) [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
10:29:53.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10:29:53.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state READY due to: [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145]
10:29:53.539 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.10.16.125:40145(127.10.16.125:40145) for tablet Kudu Master: [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145]
10:29:53.540 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.16.125:40145(127.10.16.125:40145)] handling channelInactive
10:29:53.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.548393 11460 log.cc:826] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Log is configured to *not* fsync() on all Append() calls
10:29:53.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.124:34587(127.10.16.124:34587)] connecting to peer
10:29:53.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.551 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.16.124:34587(127.10.16.124:34587)] Successfully connected to peer
10:29:53.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.551 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.553 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.16.124
10:29:53.564 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1ea3ed34, L:/127.0.0.1:40520 - R:/127.10.16.124:34587] using SASL/PLAIN
10:29:53.566 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Catalog manager is not initialized. State: Starting
10:29:53.566 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Catalog manager is not initialized. State: Starting
10:29:53.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.572 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.583657 11460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:29:53.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.584445 11460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: Bootstrap complete.
10:29:53.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Catalog manager is not initialized. State: Starting
10:29:53.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.605571 11460 raft_consensus.cc:357] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.607827 11460 raft_consensus.cc:738] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 39b21963161d48bfa33fa722af5c6c36, State: Initialized, Role: FOLLOWER
10:29:53.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.608520 11460 consensus_queue.cc:260] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:53.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.611303 11463 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:53.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.612309 11463 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.613217 11460 sys_catalog.cc:564] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: configured and running, proceeding with master startup.
10:29:53.614 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.124:34587: Catalog manager is not initialized. State: Starting
10:29:53.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.634 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.643855 11474 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.644126 11474 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: loading cluster ID for follower catalog manager: success
10:29:53.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.653554 11474 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: acquiring CA information for follower catalog manager: success
10:29:53.654 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.659554 11474 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:29:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.654261 11370 master_runner.cc:186] Error getting master registration for 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111),
10:29:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.671363 11370 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:29:53.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.711 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.10.16.125:40145, [50ms] received response from server master-127.10.16.125:40145: Network error: [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145], [50ms] delaying RPC due to: Network error: [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145], [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [64ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [77ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [77ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [125ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [125ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [143ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [164ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [165ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [222ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [223ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, deferred=Deferred@1361661817(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/333194585@606c0077 -> (continuation of Deferred@587464754 after retry RPC after error@1376980043) -> (continuation of Deferred@1239077879 after retry RPC after error@1228217603) -> (continuation of Deferred@1253278087 after retry RPC after error@572878) -> (continuation of Deferred@525505363 after retry RPC after error@2121156799) -> (continuation of Deferred@679956812 after retry RPC after error@1831104351) -> (continuation of Deferred@1718071715 after retry RPC after error@1790977632), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1704246467@785f0745 -> (continuation of Deferred@587464754 after retry RPC after error@1376980043) -> (continuation of Deferred@1239077879 after retry RPC after error@1228217603) -> (continuation of Deferred@1253278087 after retry RPC after error@572878) -> (continuation of Deferred@525505363 after retry RPC after error@2121156799) -> (continuation of Deferred@679956812 after retry RPC after error@1831104351) -> (continuation of Deferred@1718071715 after retry RPC after error@1790977632)))
10:29:53.715 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.10.16.125:40145, [50ms] received response from server master-127.10.16.125:40145: Network error: [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145], [50ms] delaying RPC due to: Network error: [peer master-127.10.16.125:40145(127.10.16.125:40145)] unexpected exception from downstream on [id: 0x13b4eeba, L:/127.0.0.1:43058 - R:/127.10.16.125:40145], [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [64ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [77ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [77ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [125ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [125ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [143ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [164ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [165ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [222ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [223ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, deferred=Deferred@1361661817(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/333194585@606c0077 -> (continuation of Deferred@587464754 after retry RPC after error@1376980043) -> (continuation of Deferred@1239077879 after retry RPC after error@1228217603) -> (continuation of Deferred@1253278087 after retry RPC after error@572878) -> (continuation of Deferred@525505363 after retry RPC after error@2121156799) -> (continuation of Deferred@679956812 after retry RPC after error@1831104351) -> (continuation of Deferred@1718071715 after retry RPC after error@1790977632), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1704246467@785f0745 -> (continuation of Deferred@587464754 after retry RPC after error@1376980043) -> (continuation of Deferred@1239077879 after retry RPC after error@1228217603) -> (continuation of Deferred@1253278087 after retry RPC after error@572878) -> (continuation of Deferred@525505363 after retry RPC after error@2121156799) -> (continuation of Deferred@679956812 after retry RPC after error@1831104351) -> (continuation of Deferred@1718071715 after retry RPC after error@1790977632)))
10:29:53.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.767654 11478 raft_consensus.cc:491] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 53d1bd87fb0148c780618938c1137c6c)
10:29:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.768071 11478 raft_consensus.cc:513] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:53.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.770397 11478 leader_election.cc:290] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:53.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.773229 11238 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:53.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.785403 11238 leader_election.cc:336] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:53.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.788669 11421 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "39b21963161d48bfa33fa722af5c6c36" is_pre_election: true
10:29:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.789733 11421 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2c6a9ec095f4488875263c4c31e1963 in term 2.
10:29:53.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.793 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.790982 11240 leader_election.cc:304] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 39b21963161d48bfa33fa722af5c6c36, a2c6a9ec095f4488875263c4c31e1963; no voters: 53d1bd87fb0148c780618938c1137c6c
10:29:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.792057 11478 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Leader pre-election won for term 4
10:29:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.792404 11478 raft_consensus.cc:491] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 53d1bd87fb0148c780618938c1137c6c)
10:29:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.792730 11478 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 3 FOLLOWER]: Advancing to term 4
10:29:53.794 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.797817 11478 raft_consensus.cc:513] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.799659 11478 leader_election.cc:290] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 4 election: Requested vote from peers 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145), 39b21963161d48bfa33fa722af5c6c36 (127.10.16.124:34587)
10:29:53.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.801218 11421 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2c6a9ec095f4488875263c4c31e1963" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "39b21963161d48bfa33fa722af5c6c36"
10:29:53.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.801806 11421 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 2 FOLLOWER]: Advancing to term 4
10:29:53.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.803942 11238 leader_election.cc:336] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111)
10:29:53.805 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.808956 11421 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2c6a9ec095f4488875263c4c31e1963 in term 4.
10:29:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.809705 11240 leader_election.cc:304] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 39b21963161d48bfa33fa722af5c6c36, a2c6a9ec095f4488875263c4c31e1963; no voters: 53d1bd87fb0148c780618938c1137c6c
10:29:53.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.810235 11478 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 4 FOLLOWER]: Leader election won for term 4
10:29:53.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.812659 11478 raft_consensus.cc:695] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [term 4 LEADER]: Becoming Leader. State: Replica: a2c6a9ec095f4488875263c4c31e1963, State: Running, Role: LEADER
10:29:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.813555 11478 consensus_queue.cc:237] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } }
10:29:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.818233 11482 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2c6a9ec095f4488875263c4c31e1963. Latest consensus state: current_term: 4 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:53.819 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.818972 11482 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: LEADER
10:29:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.821333 11484 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:29:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.826586 11484 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053]
10:29:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.827957 11484 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98]
10:29:53.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.832 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.834612 11484 tablet_loader.cc:96] loaded metadata for tablet 058b80dddb8a4fb1bae9b48f72319b08 (table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053])
10:29:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.835727 11484 tablet_loader.cc:96] loaded metadata for tablet dfd2f9dc6e1840cda5896cbdb73cdc7f (table txn_manager_fallback_rolling [id=ddb73e40ae9741ba91d634f0be6e9053])
10:29:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.836743 11484 tablet_loader.cc:96] loaded metadata for tablet e97bc8ff74e2499b8a6d0367e55f72cf (table kudu_system.kudu_transactions [id=fcc980b19dc04a3c91087664bd9b4e98])
10:29:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.837548 11484 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:29:53.838 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145; will retry
10:29:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.840631 11484 catalog_manager.cc:1260] Loaded cluster ID: 1ab14341f1214a388cecfa6e4ec50e49
10:29:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.840807 11484 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:29:53.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.844117 11484 catalog_manager.cc:1505] Loading token signing keys...
10:29:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.846879 11484 catalog_manager.cc:5910] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963: Loaded TSK: 0
10:29:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.847774 11484 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:29:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.895649 11238 consensus_peers.cc:487] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 -> Peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Couldn't send request to peer 53d1bd87fb0148c780618938c1137c6c. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:29:53.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.16.125:40145(127.10.16.125:40145)] connecting to peer
10:29:53.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.125:40145(127.10.16.125:40145)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.913 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.16.125:40145: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145
10:29:53.913 [DEBUG - pool-40-thread-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:29:53.913 [DEBUG - pool-40-thread-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [24ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [46ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [56ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [61ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [64ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [75ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [76ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [84ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [90ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [109ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [109ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [184ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, deferred=Deferred@818192227(state=PENDING, result=null, callback=(continuation of Deferred@22944127 after retry RPC after error@1756459365) -> (continuation of Deferred@1986852527 after retry RPC after error@1657229446) -> (continuation of Deferred@329459776 after retry RPC after error@1895454984) -> (continuation of Deferred@1639059510 after retry RPC after error@967393254) -> (continuation of Deferred@868413883 after retry RPC after error@299027666) -> (continuation of Deferred@1611681150 after retry RPC after error@271980693), errback=(continuation of Deferred@22944127 after retry RPC after error@1756459365) -> (continuation of Deferred@1986852527 after retry RPC after error@1657229446) -> (continuation of Deferred@329459776 after retry RPC after error@1895454984) -> (continuation of Deferred@1639059510 after retry RPC after error@967393254) -> (continuation of Deferred@868413883 after retry RPC after error@299027666) -> (continuation of Deferred@1611681150 after retry RPC after error@271980693))) after lookup
10:29:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.917424 11421 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [term 4 FOLLOWER]: Refusing update from remote peer a2c6a9ec095f4488875263c4c31e1963: Log matching property violated. Preceding OpId in replica: term: 2 index: 12. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
10:29:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:53.919175 11482 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [LEADER]: Connected to new peer: Peer: permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
10:29:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.981392 11284 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:51860 (request call id 19) took 66 ms (client timeout 66 ms). Trace:
10:29:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:53.981874 11284 rpcz_store.cc:269] 0115 10:29:53.914931 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:29:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:53.915050 (+ 119us) service_pool.cc:225] Handling call
10:29:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:29:53.981110 (+ 66060us) inbound_call.cc:173] Queueing success response
10:29:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":1808,"thread_start_us":25695,"threads_started":5}
10:29:53.984 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [24ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [46ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [56ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [61ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [64ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [75ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [76ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [84ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [90ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [109ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [109ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [184ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [185ms] sending RPC to server master-127.10.16.126:37021, [254ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@818192227(state=PENDING, result=null, callback=(continuation of Deferred@22944127 after retry RPC after error@1756459365) -> (continuation of Deferred@1986852527 after retry RPC after error@1657229446) -> (continuation of Deferred@329459776 after retry RPC after error@1895454984) -> (continuation of Deferred@1639059510 after retry RPC after error@967393254) -> (continuation of Deferred@868413883 after retry RPC after error@299027666) -> (continuation of Deferred@1611681150 after retry RPC after error@271980693) -> (continuation of Deferred@2145426723 after retry RPC@980952825), errback=(continuation of Deferred@22944127 after retry RPC after error@1756459365) -> (continuation of Deferred@1986852527 after retry RPC after error@1657229446) -> (continuation of Deferred@329459776 after retry RPC after error@1895454984) -> (continuation of Deferred@1639059510 after retry RPC after error@967393254) -> (continuation of Deferred@868413883 after retry RPC after error@299027666) -> (continuation of Deferred@1611681150 after retry RPC after error@271980693) -> (continuation of Deferred@2145426723 after retry RPC@980952825)))
10:29:53.985 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [24ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [46ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [56ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [61ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [64ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [75ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [76ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [84ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [90ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [109ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [109ms] delaying RPC due to: Service unavailable: Master config (127.10.16.124:34587,127.10.16.126:37021,127.10.16.125:40145) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.124:34587, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.126:37021, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.16.125:40145, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.126:37021: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.125:40145: Network error: Failed to connect to peer master-127.10.16.125:40145(127.10.16.125:40145): Connection refused: /127.10.16.125:40145, [184ms] Sub RPC ConnectToMaster: received response from server master-127.10.16.124:34587: OK, [185ms] sending RPC to server master-127.10.16.126:37021, [254ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@818192227(state=PENDING, result=null, callback=(continuation of Deferred@22944127 after retry RPC after error@1756459365) -> (continuation of Deferred@1986852527 after retry RPC after error@1657229446) -> (continuation of Deferred@329459776 after retry RPC after error@1895454984) -> (continuation of Deferred@1639059510 after retry RPC after error@967393254) -> (continuation of Deferred@868413883 after retry RPC after error@299027666) -> (continuation of Deferred@1611681150 after retry RPC after error@271980693) -> (continuation of Deferred@2145426723 after retry RPC@980952825), errback=(continuation of Deferred@22944127 after retry RPC after error@1756459365) -> (continuation of Deferred@1986852527 after retry RPC after error@1657229446) -> (continuation of Deferred@329459776 after retry RPC after error@1895454984) -> (continuation of Deferred@1639059510 after retry RPC after error@967393254) -> (continuation of Deferred@868413883 after retry RPC after error@299027666) -> (continuation of Deferred@1611681150 after retry RPC after error@271980693) -> (continuation of Deferred@2145426723 after retry RPC@980952825)))
10:29:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.038985 11463 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2c6a9ec095f4488875263c4c31e1963. Latest consensus state: current_term: 4 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.039752 11463 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:54.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.041124 11463 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:54.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.041806 11463 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:54.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.042344 11492 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:54.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.043110 11492 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b21963161d48bfa33fa722af5c6c36 [sys.catalog]: This master's current role is: FOLLOWER
10:29:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.061992 11482 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "a2c6a9ec095f4488875263c4c31e1963" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2c6a9ec095f4488875263c4c31e1963" member_type: VOTER last_known_addr { host: "127.10.16.126" port: 37021 } } peers { permanent_uuid: "53d1bd87fb0148c780618938c1137c6c" member_type: VOTER last_known_addr { host: "127.10.16.125" port: 40145 } } peers { permanent_uuid: "39b21963161d48bfa33fa722af5c6c36" member_type: VOTER last_known_addr { host: "127.10.16.124" port: 34587 } } }
10:29:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.062750 11482 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 [sys.catalog]: This master's current role is: LEADER
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.067478 11254 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51908:
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:29:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:29:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:29:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:29:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:29:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:29:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:54.078372 11254 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:29:54.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.152004 10963 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.009s from last keepalive heartbeat (effective timeout is 1.000s)
10:29:54.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:54.190610 11500 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
10:29:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:54.405504 11238 consensus_peers.cc:487] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 -> Peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Couldn't send request to peer 53d1bd87fb0148c780618938c1137c6c. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:29:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.429842 11081 heartbeater.cc:346] Connected to a master server at 127.10.16.124:34587
10:29:54.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.437551 11411 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e09f5e075d5b46d48a7643f6e5182995" instance_seqno: 1736936983889208) as {username='slave'} at 127.10.16.67:46581; Asking this server to re-register.
10:29:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.439232 11081 heartbeater.cc:463] Registering TS with master...
10:29:54.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.443261 11410 ts_manager.cc:194] Registered new tserver with Master: e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713)
10:29:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.453943 10933 heartbeater.cc:346] Connected to a master server at 127.10.16.124:34587
10:29:54.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.457218 11410 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9128ea62300a43dfb993e431392c642e" instance_seqno: 1736936981894189) as {username='slave'} at 127.10.16.66:36173; Asking this server to re-register.
10:29:54.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.458570 10933 heartbeater.cc:463] Registering TS with master...
10:29:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.464859 11411 ts_manager.cc:194] Registered new tserver with Master: 9128ea62300a43dfb993e431392c642e (127.10.16.66:39961)
10:29:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.501252 11411 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "2294858f518148ee8531c8bfb35cd242" instance_seqno: 1736936979208743) as {username='slave'} at 127.10.16.65:49295; Asking this server to re-register.
10:29:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.502624 10784 heartbeater.cc:463] Registering TS with master...
10:29:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.505707 11411 ts_manager.cc:194] Registered new tserver with Master: 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761)
10:29:54.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.571523 11217 master_runner.cc:186] Error getting master registration for 127.10.16.125:40145: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111),
10:29:54.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:54.571913 11217 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:29:54.656 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: e09f5e075d5b46d48a7643f6e5182995 (127.10.16.67:39713): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1736936985 last_transition_timestamp: 1736936994
10:29:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:54.886991 11238 consensus_peers.cc:487] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 -> Peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Couldn't send request to peer 53d1bd87fb0148c780618938c1137c6c. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:29:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:55.400015 11238 consensus_peers.cc:487] T 00000000000000000000000000000000 P a2c6a9ec095f4488875263c4c31e1963 -> Peer 53d1bd87fb0148c780618938c1137c6c (127.10.16.125:40145): Couldn't send request to peer 53d1bd87fb0148c780618938c1137c6c. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.125:40145: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:29:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.408097 11083 heartbeater.cc:501] Master 127.10.16.126:37021 was elected leader, sending a full tablet report...
10:29:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.459029 10935 heartbeater.cc:501] Master 127.10.16.126:37021 was elected leader, sending a full tablet report...
10:29:55.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.479871 10787 heartbeater.cc:501] Master 127.10.16.126:37021 was elected leader, sending a full tablet report...
10:29:55.542 [DEBUG - main] (Connection.java:575) [peer master-127.10.16.126:37021(127.10.16.126:37021)] disconnecting while in state READY
10:29:55.542 [DEBUG - main] (Connection.java:575) [peer master-127.10.16.125:40145(127.10.16.125:40145)] disconnecting while in state TERMINATED
10:29:55.542 [DEBUG - main] (Connection.java:575) [peer 2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)] disconnecting while in state READY
10:29:55.542 [DEBUG - main] (Connection.java:575) [peer master-127.10.16.124:34587(127.10.16.124:34587)] disconnecting while in state READY
10:29:55.542 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.16.126:37021(127.10.16.126:37021)] handling channelInactive
10:29:55.543 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.16.126:37021(127.10.16.126:37021)] cleaning up while in state READY due to: connection closed
10:29:55.546 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:29:55.549 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)] handling channelInactive
10:29:55.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2294858f518148ee8531c8bfb35cd242(127.10.16.65:41761)] cleaning up while in state READY due to: connection closed
10:29:55.550 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.16.124:34587(127.10.16.124:34587)] handling channelInactive
10:29:55.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.16.124:34587(127.10.16.124:34587)] cleaning up while in state READY due to: connection closed
10:29:55.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.547873 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 10616
10:29:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:55.591871 10958 connection.cc:537] client connection to 127.10.16.65:41761 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:29:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:55.592698 10958 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:29:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.593492 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 10789
10:29:55.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:29:55.607486 10958 consensus_peers.cc:487] T e97bc8ff74e2499b8a6d0367e55f72cf P e09f5e075d5b46d48a7643f6e5182995 -> Peer 2294858f518148ee8531c8bfb35cd242 (127.10.16.65:41761): Couldn't send request to peer 2294858f518148ee8531c8bfb35cd242. Status: Network error: Client connection negotiation failed: client connection to 127.10.16.65:41761: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:29:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.628679 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 10938
10:29:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.667742 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 11217
10:29:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:29:55.722440 10305 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskM5Ix5b/build/tsan/bin/kudu with pid 11370
10:29:56.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:29:56Z chronyd exiting
10:29:57.914 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)