09:03:22.285 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:03:22.285 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktIIAp6/test-tmp
09:03:22.285 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
09:03:22.285 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktIIAp6/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
09:03:22.286 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:03:22.620 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:03:22.620 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:03:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:22.623211 4971 mini_oidc.cc:150] Starting JWKS server
09:03:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:22.625900 4971 webserver.cc:458] Webserver started at http://127.0.0.1:35251/ using document root <none> and password file <none>
09:03:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:22.626120 4971 mini_oidc.cc:186] Starting OIDC Discovery server
09:03:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:22.626722 4971 webserver.cc:458] Webserver started at http://127.0.0.1:34417/ using document root <none> and password file <none>
09:03:22.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-11T09:03:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:03:22.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-11T09:03:22Z Disabled control of system clock
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:22.659081 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/logs
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data/info.pb
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.254:46469
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.254
09:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:03:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:22.998282 4983 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:22.998620 4983 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:22.998720 4983 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.006819 4983 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.006947 4983 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.006987 4983 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.007037 4983 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.007078 4983 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.007130 4983 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.007164 4983 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35251/jwks.json
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.017068 4983 master_runner.cc:386] Master server non-default flags:
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.254:46469
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data/info.pb
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.254
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/logs
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.018724 4983 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.020478 4983 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:23.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.031059 4988 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.031792 4989 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.032253 4991 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.032992 4983 server_base.cc:1034] running on GCE node
09:03:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.033852 4983 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:23.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.035414 4983 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.036739 4983 hybrid_clock.cc:648] HybridClock initialized: now 1736586203036696 us; error 69 us; skew 500 ppm
09:03:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.037083 4983 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.039438 4983 webserver.cc:458] Webserver started at http://127.4.218.254:37885/ using document root <none> and password file <none>
09:03:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.040237 4983 fs_manager.cc:362] Metadata directory not provided
09:03:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.040333 4983 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.040616 4983 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.043316 4983 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data/instance:
09:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb6fc2babbb4eba82e1a977da112d45"
09:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.044203 4983 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal/instance:
09:03:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb6fc2babbb4eba82e1a977da112d45"
09:03:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.049681 4983 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
09:03:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.053344 4998 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:23.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.054814 4983 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.054981 4983 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb6fc2babbb4eba82e1a977da112d45"
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.055173 4983 fs_report.cc:389] FS layout report
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data/data
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.089141 4983 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.090008 4983 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.090258 4983 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.111689 4983 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.254:46469
09:03:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.111750 5059 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.254:46469 every 8 connection(s)
09:03:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.113049 4983 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/data/info.pb
09:03:23.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.116915 5060 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.117874 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 4983
09:03:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.118187 4971 external_mini_cluster.cc:1427] Reading /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-0/wal/instance
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.121186 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/logs
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data/info.pb
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.253:40603
09:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.253
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:03:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:03:23.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.126194 5060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } has no permanent_uuid. Determining permanent_uuid...
09:03:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.142783 5060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } has no permanent_uuid. Determining permanent_uuid...
09:03:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.143927 4999 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111)
09:03:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.145438 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.253:40603: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111)
09:03:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.194867 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } attempt: 1
09:03:23.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.196856 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.253:40603: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111)
09:03:23.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.265121 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } attempt: 2
09:03:23.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.266945 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.253:40603: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111)
09:03:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.358228 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } attempt: 3
09:03:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.360283 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.253:40603: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111)
09:03:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.460007 5062 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.460350 5062 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.460502 5062 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.468591 5062 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:03:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.468744 5062 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:03:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.468806 5062 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.468883 5062 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.468945 5062 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:03:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.469026 5062 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.469099 5062 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35251/jwks.json
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.479009 5062 master_runner.cc:386] Master server non-default flags:
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.253:40603
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data/info.pb
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.253
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/logs
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.481106 5062 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.482961 5062 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:23.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.493567 5070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.494351 5071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.495018 5073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.495857 5062 server_base.cc:1034] running on GCE node
09:03:23.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.496748 5062 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.498314 5062 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.499555 5062 hybrid_clock.cc:648] HybridClock initialized: now 1736586203499501 us; error 35 us; skew 500 ppm
09:03:23.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.500038 5062 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.502612 5062 webserver.cc:458] Webserver started at http://127.4.218.253:43177/ using document root <none> and password file <none>
09:03:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.503408 5062 fs_manager.cc:362] Metadata directory not provided
09:03:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.503525 5062 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.503512 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } attempt: 4
09:03:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.503950 5062 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:03:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.505384 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.253:40603: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111)
09:03:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.506968 5062 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data/instance:
09:03:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.507910 5062 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal/instance:
09:03:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.513383 5062 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
09:03:23.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.517032 5079 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:23.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.518433 5062 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:03:23.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.518644 5062 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal
09:03:23.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:23.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.518877 5062 fs_report.cc:389] FS layout report
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data/data
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.548934 5062 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.549892 5062 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.550199 5062 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.571892 5062 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.253:40603
09:03:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.571928 5140 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.253:40603 every 8 connection(s)
09:03:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.573374 5062 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/data/info.pb
09:03:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.577426 5141 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:23.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.581508 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 5062
09:03:23.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.581725 4971 external_mini_cluster.cc:1427] Reading /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-1/wal/instance
09:03:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.582767 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/logs
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/info.pb
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.252:35957
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.252
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:03:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:03:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.583735 5141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } has no permanent_uuid. Determining permanent_uuid...
09:03:23.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.598054 5141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } has no permanent_uuid. Determining permanent_uuid...
09:03:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.607856 5141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } has no permanent_uuid. Determining permanent_uuid...
09:03:23.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.609130 5082 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.610646 5141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.659969 5141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 1
09:03:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.661944 5141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.730192 5141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 2
09:03:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.732148 5141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.804692 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } attempt: 5
09:03:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.811647 5060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } has no permanent_uuid. Determining permanent_uuid...
09:03:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.813923 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.823405 5141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 3
09:03:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.825453 5141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.865181 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 1
09:03:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.867120 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.935359 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 2
09:03:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.937482 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.941414 5143 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:23.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.941733 5143 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:23.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.941886 5143 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.949901 5143 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.950050 5143 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.950107 5143 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.950176 5143 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.950240 5143 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.950320 5143 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:23.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.950394 5143 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35251/jwks.json
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.961127 5143 master_runner.cc:386] Master server non-default flags:
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.252:35957
09:03:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/info.pb
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.252
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/logs
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.963164 5143 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.964926 5143 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:23.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.968678 5141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 4
09:03:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.970708 5141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.976173 5151 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.976951 5154 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:23.977542 5152 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.977989 5143 server_base.cc:1034] running on GCE node
09:03:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.978806 5143 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.980363 5143 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.981568 5143 hybrid_clock.cc:648] HybridClock initialized: now 1736586203981523 us; error 36 us; skew 500 ppm
09:03:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.981899 5143 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:23.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.989567 5143 webserver.cc:458] Webserver started at http://127.4.218.252:41581/ using document root <none> and password file <none>
09:03:23.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.990412 5143 fs_manager.cc:362] Metadata directory not provided
09:03:23.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.990528 5143 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:23.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.990864 5143 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:03:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.993635 5143 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/instance:
09:03:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51ede745e8b42f4b48d705dc3953e5a"
09:03:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.994392 5143 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal/instance:
09:03:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51ede745e8b42f4b48d705dc3953e5a"
09:03:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:23.999749 5143 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.005s
09:03:24.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.003198 5160 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.004626 5143 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:03:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.004776 5143 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51ede745e8b42f4b48d705dc3953e5a"
09:03:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.004958 5143 fs_report.cc:389] FS layout report
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/data
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:24.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.029399 5143 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:24.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.030244 5143 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:24.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.030481 5143 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.043702 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 3
09:03:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.045907 5060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.052094 5143 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.252:35957
09:03:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.052114 5221 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.252:35957 every 8 connection(s)
09:03:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.053459 5143 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/info.pb
09:03:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.057228 5222 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.062433 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 5143
09:03:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.062631 4971 external_mini_cluster.cc:1427] Reading /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal/instance
09:03:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.063768 5222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } has no permanent_uuid. Determining permanent_uuid...
09:03:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.078440 5222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } has no permanent_uuid. Determining permanent_uuid...
09:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.087332 5222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } has no permanent_uuid. Determining permanent_uuid...
09:03:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.098340 5222 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Bootstrap starting.
09:03:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.101866 5222 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Neither blocks nor log segments found. Creating new log.
09:03:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.103004 5222 log.cc:826] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Log is configured to *not* fsync() on all Append() calls
09:03:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.105891 5222 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: No bootstrap required, opened a new log
09:03:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.113512 5222 raft_consensus.cc:357] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.113900 5222 raft_consensus.cc:383] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.113998 5222 raft_consensus.cc:738] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f51ede745e8b42f4b48d705dc3953e5a, State: Initialized, Role: FOLLOWER
09:03:24.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.114724 5222 consensus_queue.cc:260] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.115846 5231 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.116240 5231 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.117180 5222 sys_catalog.cc:564] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: configured and running, proceeding with master startup.
09:03:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.129736 5242 catalog_manager.cc:1559] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.129925 5242 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.124701 5143 master_runner.cc:186] Error getting master registration for 127.4.218.254:46469: OK, instance_id { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" instance_seqno: 1736586203106737 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.131966 5143 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:03:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.191156 5231 raft_consensus.cc:491] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.191474 5231 raft_consensus.cc:513] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.193413 5231 leader_election.cc:290] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603)
09:03:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.223110 5060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 4
09:03:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.235453 5060 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: Bootstrap starting.
09:03:24.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.239962 5060 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: Neither blocks nor log segments found. Creating new log.
09:03:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.241899 5060 log.cc:826] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: Log is configured to *not* fsync() on all Append() calls
09:03:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.245237 5060 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: No bootstrap required, opened a new log
09:03:24.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.254876 5060 raft_consensus.cc:357] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.255291 5060 raft_consensus.cc:383] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:24.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.255407 5060 raft_consensus.cc:738] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: feb6fc2babbb4eba82e1a977da112d45, State: Initialized, Role: FOLLOWER
09:03:24.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.256215 5060 consensus_queue.cc:260] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.257484 5245 sys_catalog.cc:455] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.257973 5245 sys_catalog.cc:458] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.258682 5060 sys_catalog.cc:564] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: configured and running, proceeding with master startup.
09:03:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.258747 5025 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f51ede745e8b42f4b48d705dc3953e5a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "feb6fc2babbb4eba82e1a977da112d45" is_pre_election: true
09:03:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.259364 5025 raft_consensus.cc:2463] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f51ede745e8b42f4b48d705dc3953e5a in term 0.
09:03:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.260404 5164 leader_election.cc:304] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: f51ede745e8b42f4b48d705dc3953e5a, feb6fc2babbb4eba82e1a977da112d45; no voters:
09:03:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.260905 5231 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.261150 5231 raft_consensus.cc:491] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.261273 5231 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 0 FOLLOWER]: Advancing to term 1
09:03:24.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.264506 5231 raft_consensus.cc:513] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.267146 5025 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f51ede745e8b42f4b48d705dc3953e5a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "feb6fc2babbb4eba82e1a977da112d45"
09:03:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.267381 5025 raft_consensus.cc:3054] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 0 FOLLOWER]: Advancing to term 1
09:03:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.267536 5231 leader_election.cc:290] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 1 election: Requested vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603)
09:03:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.269938 5141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } attempt: 5
09:03:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.270748 5025 raft_consensus.cc:2463] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f51ede745e8b42f4b48d705dc3953e5a in term 1.
09:03:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.271497 5164 leader_election.cc:304] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: f51ede745e8b42f4b48d705dc3953e5a, feb6fc2babbb4eba82e1a977da112d45; no voters:
09:03:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.271893 5231 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 1 FOLLOWER]: Leader election won for term 1
09:03:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.282574 5166 tablet.cc:2367] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:24.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.287941 5141 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: Bootstrap starting.
09:03:24.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.288684 5231 raft_consensus.cc:695] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 1 LEADER]: Becoming Leader. State: Replica: f51ede745e8b42f4b48d705dc3953e5a, State: Running, Role: LEADER
09:03:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.290149 5231 consensus_queue.cc:237] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.293215 5141 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: Neither blocks nor log segments found. Creating new log.
09:03:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.294765 5141 log.cc:826] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: Log is configured to *not* fsync() on all Append() calls
09:03:24.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.305805 5251 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: New leader f51ede745e8b42f4b48d705dc3953e5a. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.306308 5251 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: LEADER
09:03:24.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.308795 5141 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: No bootstrap required, opened a new log
09:03:24.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.320034 5258 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:03:24.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.323037 5258 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:03:24.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.330780 5141 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.331692 5141 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.332016 5141 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7549c035609942c2a1e7a1a6e1fc884f, State: Initialized, Role: FOLLOWER
09:03:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.333518 5141 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:24.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.328042 4983 master_runner.cc:186] Error getting master registration for 127.4.218.253:40603: OK, instance_id { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" instance_seqno: 1736586203566522 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.338537 4983 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:03:24.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.341877 5025 raft_consensus.cc:1270] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Refusing update from remote peer f51ede745e8b42f4b48d705dc3953e5a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:03:24.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.343127 5004 tablet.cc:2367] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:24.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.343310 5251 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [LEADER]: Connected to new peer: Peer: permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.348991 5105 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f51ede745e8b42f4b48d705dc3953e5a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:24.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.349498 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f51ede745e8b42f4b48d705dc3953e5a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f" is_pre_election: true
09:03:24.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.350214 5141 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: configured and running, proceeding with master startup.
09:03:24.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.350608 5104 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 0 FOLLOWER]: Advancing to term 1
09:03:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.352558 5260 catalog_manager.cc:1559] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.352794 5260 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:24.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.353943 5104 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Refusing update from remote peer f51ede745e8b42f4b48d705dc3953e5a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:03:24.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.355232 5251 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.359184 5105 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f51ede745e8b42f4b48d705dc3953e5a in term 1.
09:03:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.359956 5262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.360386 5262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.388389 5262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f51ede745e8b42f4b48d705dc3953e5a. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.388792 5262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.389258 5245 sys_catalog.cc:455] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f51ede745e8b42f4b48d705dc3953e5a. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.389719 5245 sys_catalog.cc:458] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.394800 5251 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.395171 5251 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: LEADER
09:03:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.396050 5258 catalog_manager.cc:1348] Generated new cluster ID: 3d8e045736ab427bbdcb73e0d007dcad
09:03:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.396169 5258 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:03:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.401880 5231 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.402348 5231 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: LEADER
09:03:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.405064 5262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.405328 5262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.419659 5264 mvcc.cc:204] Tried to move back new op lower bound from 7113057092964962304 to 7113057092788924416. Current Snapshot: MvccSnapshot[applied={T|T < 7113057092964962304}]
09:03:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.426009 5245 sys_catalog.cc:455] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.426276 5245 sys_catalog.cc:458] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: This master's current role is: FOLLOWER
09:03:24.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.464932 5258 catalog_manager.cc:1371] Generated new certificate authority record
09:03:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.468910 5258 catalog_manager.cc:1505] Loading token signing keys...
09:03:24.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.472790 5282 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.511391 5258 catalog_manager.cc:5899] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Generated new TSK 0
09:03:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.513073 5258 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:03:24.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.542685 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/logs
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data/info.pb
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.193:0
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.218.193
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.193
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:03:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:03:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.948246 5283 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.948549 5283 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.948659 5283 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.957839 5283 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.958006 5283 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:03:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.958045 5283 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:03:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.958091 5283 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:03:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.958181 5283 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.218.193
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.968046 5283 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.193:0
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data/info.pb
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.193
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.218.193
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/logs
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.969663 5283 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:24.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.971371 5283 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:24.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.984241 5289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:24.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.984884 5291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:24.985877 5288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.986583 5283 server_base.cc:1034] running on GCE node
09:03:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.987221 5283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.988301 5283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.989540 5283 hybrid_clock.cc:648] HybridClock initialized: now 1736586204989466 us; error 60 us; skew 500 ppm
09:03:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.989905 5283 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.992341 5283 webserver.cc:458] Webserver started at http://127.4.218.193:46543/ using document root <none> and password file <none>
09:03:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.993172 5283 fs_manager.cc:362] Metadata directory not provided
09:03:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.993292 5283 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.993639 5283 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:03:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.996414 5283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data/instance:
09:03:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9311c63c2b94110958d997d35d1e689"
09:03:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:24 on dist-test-slave-cspg"
09:03:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:24.997280 5283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal/instance:
09:03:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9311c63c2b94110958d997d35d1e689"
09:03:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:24 on dist-test-slave-cspg"
09:03:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.002765 5283 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
09:03:25.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.006388 5297 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.007877 5283 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.008064 5283 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9311c63c2b94110958d997d35d1e689"
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:24 on dist-test-slave-cspg"
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.008270 5283 fs_report.cc:389] FS layout report
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data/data
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.032944 5283 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.033859 5283 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:25.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.034116 5283 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.037246 5283 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.037339 5283 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.037442 5283 ts_tablet_manager.cc:610] Registered 0 tablets
09:03:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.037503 5283 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.140986 5421 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.193:33309 every 8 connection(s)
09:03:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.140980 5283 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.193:33309
09:03:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.142697 5283 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/data/info.pb
09:03:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.145828 5423 heartbeater.cc:346] Connected to a master server at 127.4.218.254:46469
09:03:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.146124 5423 heartbeater.cc:463] Registering TS with master...
09:03:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.148531 5423 heartbeater.cc:510] Master 127.4.218.254:46469 requested a full tablet report, sending...
09:03:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.151958 5015 ts_manager.cc:194] Registered new tserver with Master: e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309)
09:03:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.152890 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 5283
09:03:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.153066 4971 external_mini_cluster.cc:1427] Reading /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-0/wal/instance
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.157341 5422 heartbeater.cc:346] Connected to a master server at 127.4.218.252:35957
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.157491 5422 heartbeater.cc:463] Registering TS with master...
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.157627 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/logs
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data/info.pb
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.194:0
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.218.194
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.194
09:03:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.158070 5422 heartbeater.cc:510] Master 127.4.218.252:35957 requested a full tablet report, sending...
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.159541 5424 heartbeater.cc:346] Connected to a master server at 127.4.218.253:40603
09:03:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.159732 5424 heartbeater.cc:463] Registering TS with master...
09:03:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.159950 5177 ts_manager.cc:194] Registered new tserver with Master: e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309)
09:03:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.160256 5424 heartbeater.cc:510] Master 127.4.218.253:40603 requested a full tablet report, sending...
09:03:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.161787 5096 ts_manager.cc:194] Registered new tserver with Master: e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309)
09:03:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.162132 5177 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.218.193:39279
09:03:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.355983 5260 catalog_manager.cc:1260] Loaded cluster ID: 3d8e045736ab427bbdcb73e0d007dcad
09:03:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.356125 5260 catalog_manager.cc:1553] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: loading cluster ID for follower catalog manager: success
09:03:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.360105 5260 catalog_manager.cc:1575] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: acquiring CA information for follower catalog manager: success
09:03:25.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.362790 5260 catalog_manager.cc:1603] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.478714 5282 catalog_manager.cc:1260] Loaded cluster ID: 3d8e045736ab427bbdcb73e0d007dcad
09:03:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.478889 5282 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: loading cluster ID for follower catalog manager: success
09:03:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.483359 5282 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: acquiring CA information for follower catalog manager: success
09:03:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.486188 5282 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.502192 5427 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.502502 5427 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.502617 5427 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.511440 5427 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.511663 5427 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:03:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.511736 5427 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:03:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.511803 5427 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:03:25.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.511938 5427 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.218.194
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.522212 5427 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.194:0
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data/info.pb
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.194
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:03:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.218.194
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/logs
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.524315 5427 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:25.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.526120 5427 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.539148 5434 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.539506 5433 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:25.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:25.539917 5436 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:25.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.541530 5427 server_base.cc:1034] running on GCE node
09:03:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.542309 5427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.543519 5427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.544929 5427 hybrid_clock.cc:648] HybridClock initialized: now 1736586205544805 us; error 102 us; skew 500 ppm
09:03:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.545339 5427 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.548295 5427 webserver.cc:458] Webserver started at http://127.4.218.194:44177/ using document root <none> and password file <none>
09:03:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.549312 5427 fs_manager.cc:362] Metadata directory not provided
09:03:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.549440 5427 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.549810 5427 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.553485 5427 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data/instance:
09:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698e8d4afb8f46c08c4075b41d506c40"
09:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:25 on dist-test-slave-cspg"
09:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.554489 5427 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal/instance:
09:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698e8d4afb8f46c08c4075b41d506c40"
09:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:25 on dist-test-slave-cspg"
09:03:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.561523 5427 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
09:03:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.566063 5442 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.567893 5427 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.568071 5427 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698e8d4afb8f46c08c4075b41d506c40"
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:25 on dist-test-slave-cspg"
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.568343 5427 fs_report.cc:389] FS layout report
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data/data
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.600582 5427 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.601745 5427 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.602082 5427 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.605924 5427 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.606213 5427 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
09:03:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.606362 5427 ts_tablet_manager.cc:610] Registered 0 tablets
09:03:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.606428 5427 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:25.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.697079 5427 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.194:42447
09:03:25.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.697103 5565 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.194:42447 every 8 connection(s)
09:03:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.698446 5427 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/data/info.pb
09:03:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.702471 5567 heartbeater.cc:346] Connected to a master server at 127.4.218.254:46469
09:03:25.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.703332 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 5427
09:03:25.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.703541 4971 external_mini_cluster.cc:1427] Reading /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-1/wal/instance
09:03:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.708716 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/logs
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data/info.pb
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.195:0
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.218.195
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.195
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:03:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:03:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:03:25.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.717288 5566 heartbeater.cc:346] Connected to a master server at 127.4.218.252:35957
09:03:25.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.717770 5566 heartbeater.cc:463] Registering TS with master...
09:03:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.719040 5566 heartbeater.cc:510] Master 127.4.218.252:35957 requested a full tablet report, sending...
09:03:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.720902 5177 ts_manager.cc:194] Registered new tserver with Master: 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447)
09:03:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.722026 5177 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.218.194:54447
09:03:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.718006 5567 heartbeater.cc:463] Registering TS with master...
09:03:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.724303 5567 heartbeater.cc:510] Master 127.4.218.254:46469 requested a full tablet report, sending...
09:03:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.723132 5568 heartbeater.cc:346] Connected to a master server at 127.4.218.253:40603
09:03:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.725111 5568 heartbeater.cc:463] Registering TS with master...
09:03:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.725572 5568 heartbeater.cc:510] Master 127.4.218.253:40603 requested a full tablet report, sending...
09:03:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.725697 5015 ts_manager.cc:194] Registered new tserver with Master: 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447)
09:03:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:25.727270 5096 ts_manager.cc:194] Registered new tserver with Master: 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447)
09:03:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.060251 5571 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.060657 5571 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.060832 5571 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.071527 5571 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.071756 5571 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:03:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.071802 5571 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:03:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.071851 5571 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:03:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.071943 5571 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.218.195
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.081784 5571 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.195:0
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data/info.pb
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.195
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.218.195
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/logs
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.083398 5571 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.085211 5571 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:26.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.097191 5576 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.097926 5577 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.098583 5579 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.098979 5571 server_base.cc:1034] running on GCE node
09:03:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.099545 5571 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.100510 5571 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.101743 5571 hybrid_clock.cc:648] HybridClock initialized: now 1736586206101695 us; error 50 us; skew 500 ppm
09:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.102082 5571 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:26.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.104358 5571 webserver.cc:458] Webserver started at http://127.4.218.195:46025/ using document root <none> and password file <none>
09:03:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.105103 5571 fs_manager.cc:362] Metadata directory not provided
09:03:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.105204 5571 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.105504 5571 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:03:26.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.108209 5571 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data/instance:
09:03:26.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9be3bc22b32c4eb6bae392ab7519b984"
09:03:26.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:26 on dist-test-slave-cspg"
09:03:26.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.109026 5571 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal/instance:
09:03:26.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9be3bc22b32c4eb6bae392ab7519b984"
09:03:26.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:26 on dist-test-slave-cspg"
09:03:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.114833 5571 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
09:03:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.118767 5585 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.120239 5571 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.120399 5571 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9be3bc22b32c4eb6bae392ab7519b984"
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:26 on dist-test-slave-cspg"
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.120616 5571 fs_report.cc:389] FS layout report
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data/data
09:03:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.143981 5571 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:26.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.144860 5571 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:26.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.145165 5571 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.148149 5571 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.148252 5571 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.148366 5571 ts_tablet_manager.cc:610] Registered 0 tablets
09:03:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.148440 5571 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.164700 5422 heartbeater.cc:502] Master 127.4.218.252:35957 was elected leader, sending a full tablet report...
09:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.209370 5571 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.195:39399
09:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.209406 5708 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.195:39399 every 8 connection(s)
09:03:26.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.210666 5571 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/data/info.pb
09:03:26.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.213698 5710 heartbeater.cc:346] Connected to a master server at 127.4.218.254:46469
09:03:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.214073 5710 heartbeater.cc:463] Registering TS with master...
09:03:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.215334 5710 heartbeater.cc:510] Master 127.4.218.254:46469 requested a full tablet report, sending...
09:03:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.217168 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 5571
09:03:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.217406 4971 external_mini_cluster.cc:1427] Reading /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/ts-2/wal/instance
09:03:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.217619 5015 ts_manager.cc:194] Registered new tserver with Master: 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:26.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.225847 5709 heartbeater.cc:346] Connected to a master server at 127.4.218.252:35957
09:03:26.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.226012 5709 heartbeater.cc:463] Registering TS with master...
09:03:26.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.226545 5709 heartbeater.cc:510] Master 127.4.218.252:35957 requested a full tablet report, sending...
09:03:26.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.227499 5177 ts_manager.cc:194] Registered new tserver with Master: 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.228583 5177 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.218.195:55201
09:03:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.231916 5711 heartbeater.cc:346] Connected to a master server at 127.4.218.253:40603
09:03:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.232050 5711 heartbeater.cc:463] Registering TS with master...
09:03:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.232347 5711 heartbeater.cc:510] Master 127.4.218.253:40603 requested a full tablet report, sending...
09:03:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.233366 5096 ts_manager.cc:194] Registered new tserver with Master: 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:26.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.243286 4971 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
09:03:26.245 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:03:26.245 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:03:26.246 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.218.254"
port: 46469
}
bound_http_address {
host: "127.4.218.254"
port: 37885
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.218.253"
port: 40603
}
bound_http_address {
host: "127.4.218.253"
port: 43177
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.218.252"
port: 35957
}
bound_http_address {
host: "127.4.218.252"
port: 41581
}
}
}
09:03:26.246 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:03:26.246 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.218.193"
port: 33309
}
bound_http_address {
host: "127.4.218.193"
port: 46543
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.218.194"
port: 42447
}
bound_http_address {
host: "127.4.218.194"
port: 44177
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.218.195"
port: 39399
}
bound_http_address {
host: "127.4.218.195"
port: 46025
}
}
}
09:03:26.246 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:03:26.254 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:03:26.255 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:26.256 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:26.257 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.218.252:35957(127.4.218.252:35957)] Successfully connected to peer
09:03:26.258 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.218.253:40603(127.4.218.253:40603)] connecting to peer
09:03:26.259 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.218.254:46469(127.4.218.254:46469)] Successfully connected to peer
09:03:26.261 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.218.252
09:03:26.261 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.218.254
09:03:26.263 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.218.253:40603(127.4.218.253:40603)] Successfully connected to peer
09:03:26.265 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.218.253
09:03:26.277 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa36cb118, L:/127.0.0.1:60580 - R:/127.4.218.254:46469] using SASL/PLAIN
09:03:26.278 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8887007e, L:/127.0.0.1:50150 - R:/127.4.218.253:40603] using SASL/PLAIN
09:03:26.279 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0f5e4505, L:/127.0.0.1:39686 - R:/127.4.218.252:35957] using SASL/PLAIN
09:03:26.282 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:03:26.282 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=27), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [25ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, deferred=null) after lookup
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.284370 5177 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39686:
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:03:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
09:03:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.286783 5177 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:03:26.308 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 70f8350d8f6f45cd804021cac5ee71c2
09:03:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.353128 5356 tablet_service.cc:1467] Processing CreateTablet for tablet 25c77200f9214188b0e9e0a302f614ff (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:03:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.353086 5355 tablet_service.cc:1467] Processing CreateTablet for tablet 198c32aa056e4b82adbcf0e0bd4caac0 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:03:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.355095 5356 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 25c77200f9214188b0e9e0a302f614ff. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.355568 5355 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 198c32aa056e4b82adbcf0e0bd4caac0. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.360548 5499 tablet_service.cc:1467] Processing CreateTablet for tablet 198c32aa056e4b82adbcf0e0bd4caac0 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:03:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.360695 5642 tablet_service.cc:1467] Processing CreateTablet for tablet 25c77200f9214188b0e9e0a302f614ff (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:03:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.361913 5499 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 198c32aa056e4b82adbcf0e0bd4caac0. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.361923 5642 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 25c77200f9214188b0e9e0a302f614ff. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.360030 5500 tablet_service.cc:1467] Processing CreateTablet for tablet 25c77200f9214188b0e9e0a302f614ff (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:03:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.368662 5641 tablet_service.cc:1467] Processing CreateTablet for tablet 198c32aa056e4b82adbcf0e0bd4caac0 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:03:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.369282 5641 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 198c32aa056e4b82adbcf0e0bd4caac0. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.371912 5500 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 25c77200f9214188b0e9e0a302f614ff. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.382237 5735 tablet_bootstrap.cc:492] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: Bootstrap starting.
09:03:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.385835 5735 tablet_bootstrap.cc:654] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: Neither blocks nor log segments found. Creating new log.
09:03:26.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.386818 5738 tablet_bootstrap.cc:492] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40: Bootstrap starting.
09:03:26.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.387207 5735 log.cc:826] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: Log is configured to *not* fsync() on all Append() calls
09:03:26.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.387725 5737 tablet_bootstrap.cc:492] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984: Bootstrap starting.
09:03:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.390290 5735 tablet_bootstrap.cc:492] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: No bootstrap required, opened a new log
09:03:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.390545 5737 tablet_bootstrap.cc:654] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984: Neither blocks nor log segments found. Creating new log.
09:03:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.390599 5735 ts_tablet_manager.cc:1397] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
09:03:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.390772 5738 tablet_bootstrap.cc:654] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40: Neither blocks nor log segments found. Creating new log.
09:03:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.391657 5737 log.cc:826] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984: Log is configured to *not* fsync() on all Append() calls
09:03:26.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.392066 5738 log.cc:826] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40: Log is configured to *not* fsync() on all Append() calls
09:03:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.394292 5737 tablet_bootstrap.cc:492] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984: No bootstrap required, opened a new log
09:03:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.394562 5737 ts_tablet_manager.cc:1397] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
09:03:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.394901 5738 tablet_bootstrap.cc:492] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40: No bootstrap required, opened a new log
09:03:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.395156 5738 ts_tablet_manager.cc:1397] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
09:03:26.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.399564 5735 raft_consensus.cc:357] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.400046 5735 raft_consensus.cc:383] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:26.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.400159 5735 raft_consensus.cc:738] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9311c63c2b94110958d997d35d1e689, State: Initialized, Role: FOLLOWER
09:03:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.400862 5735 consensus_queue.cc:260] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.401315 5737 raft_consensus.cc:357] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.401751 5737 raft_consensus.cc:383] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.401860 5737 raft_consensus.cc:738] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9be3bc22b32c4eb6bae392ab7519b984, State: Initialized, Role: FOLLOWER
09:03:26.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.402477 5737 consensus_queue.cc:260] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.402720 5735 ts_tablet_manager.cc:1428] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: Time spent starting tablet: real 0.012s user 0.010s sys 0.004s
09:03:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.402961 5425 tablet.cc:2367] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.403841 5735 tablet_bootstrap.cc:492] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689: Bootstrap starting.
09:03:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.405566 5735 tablet_bootstrap.cc:654] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689: Neither blocks nor log segments found. Creating new log.
09:03:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.406275 5738 raft_consensus.cc:357] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.406670 5738 raft_consensus.cc:383] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.406809 5738 raft_consensus.cc:738] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 698e8d4afb8f46c08c4075b41d506c40, State: Initialized, Role: FOLLOWER
09:03:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.407698 5738 consensus_queue.cc:260] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.414377 5735 tablet_bootstrap.cc:492] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689: No bootstrap required, opened a new log
09:03:26.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.414570 5735 ts_tablet_manager.cc:1397] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.007s
09:03:26.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.415429 5735 raft_consensus.cc:357] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.415666 5735 raft_consensus.cc:383] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.415756 5735 raft_consensus.cc:738] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9311c63c2b94110958d997d35d1e689, State: Initialized, Role: FOLLOWER
09:03:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.416007 5735 consensus_queue.cc:260] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.416296 5738 ts_tablet_manager.cc:1428] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40: Time spent starting tablet: real 0.021s user 0.014s sys 0.005s
09:03:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.416633 5566 heartbeater.cc:502] Master 127.4.218.252:35957 was elected leader, sending a full tablet report...
09:03:26.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.420908 5735 ts_tablet_manager.cc:1428] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689: Time spent starting tablet: real 0.006s user 0.000s sys 0.007s
09:03:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.423632 5738 tablet_bootstrap.cc:492] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40: Bootstrap starting.
09:03:26.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.425513 5738 tablet_bootstrap.cc:654] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40: Neither blocks nor log segments found. Creating new log.
09:03:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.430290 5709 heartbeater.cc:502] Master 127.4.218.252:35957 was elected leader, sending a full tablet report...
09:03:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.431207 5737 ts_tablet_manager.cc:1428] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984: Time spent starting tablet: real 0.037s user 0.018s sys 0.014s
09:03:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.431687 5737 tablet_bootstrap.cc:492] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984: Bootstrap starting.
09:03:26.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.433625 5737 tablet_bootstrap.cc:654] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984: Neither blocks nor log segments found. Creating new log.
09:03:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.436074 5738 tablet_bootstrap.cc:492] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40: No bootstrap required, opened a new log
09:03:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.436244 5738 ts_tablet_manager.cc:1397] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.004s
09:03:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.437021 5738 raft_consensus.cc:357] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.437182 5738 raft_consensus.cc:383] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.437268 5738 raft_consensus.cc:738] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 698e8d4afb8f46c08c4075b41d506c40, State: Initialized, Role: FOLLOWER
09:03:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.437541 5737 tablet_bootstrap.cc:492] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984: No bootstrap required, opened a new log
09:03:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.437512 5738 consensus_queue.cc:260] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.437664 5737 ts_tablet_manager.cc:1397] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
09:03:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.438200 5738 ts_tablet_manager.cc:1428] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
09:03:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.438446 5737 raft_consensus.cc:357] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.438652 5737 raft_consensus.cc:383] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.438746 5737 raft_consensus.cc:738] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9be3bc22b32c4eb6bae392ab7519b984, State: Initialized, Role: FOLLOWER
09:03:26.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.438998 5737 consensus_queue.cc:260] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.439541 5737 ts_tablet_manager.cc:1428] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
09:03:26.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.445730 5743 raft_consensus.cc:491] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.445937 5743 raft_consensus.cc:513] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.447489 5743 leader_election.cc:290] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309), 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447)
09:03:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.455278 5520 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "25c77200f9214188b0e9e0a302f614ff" candidate_uuid: "9be3bc22b32c4eb6bae392ab7519b984" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "698e8d4afb8f46c08c4075b41d506c40" is_pre_election: true
09:03:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.455278 5377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "25c77200f9214188b0e9e0a302f614ff" candidate_uuid: "9be3bc22b32c4eb6bae392ab7519b984" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9311c63c2b94110958d997d35d1e689" is_pre_election: true
09:03:26.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.455909 5377 raft_consensus.cc:2463] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9be3bc22b32c4eb6bae392ab7519b984 in term 0.
09:03:26.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.455899 5520 raft_consensus.cc:2463] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9be3bc22b32c4eb6bae392ab7519b984 in term 0.
09:03:26.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.456569 5586 leader_election.cc:304] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 698e8d4afb8f46c08c4075b41d506c40, 9be3bc22b32c4eb6bae392ab7519b984, e9311c63c2b94110958d997d35d1e689; no voters:
09:03:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.456959 5743 raft_consensus.cc:2798] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.457089 5743 raft_consensus.cc:491] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.457156 5743 raft_consensus.cc:3054] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Advancing to term 1
09:03:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.459079 5743 raft_consensus.cc:513] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.459758 5743 leader_election.cc:290] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [CANDIDATE]: Term 1 election: Requested vote from peers e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309), 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447)
09:03:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.460130 5377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "25c77200f9214188b0e9e0a302f614ff" candidate_uuid: "9be3bc22b32c4eb6bae392ab7519b984" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9311c63c2b94110958d997d35d1e689"
09:03:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.460254 5520 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "25c77200f9214188b0e9e0a302f614ff" candidate_uuid: "9be3bc22b32c4eb6bae392ab7519b984" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "698e8d4afb8f46c08c4075b41d506c40"
09:03:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.460345 5377 raft_consensus.cc:3054] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Advancing to term 1
09:03:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.460510 5520 raft_consensus.cc:3054] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Advancing to term 1
09:03:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.462405 5377 raft_consensus.cc:2463] T 25c77200f9214188b0e9e0a302f614ff P e9311c63c2b94110958d997d35d1e689 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9be3bc22b32c4eb6bae392ab7519b984 in term 1.
09:03:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.462513 5520 raft_consensus.cc:2463] T 25c77200f9214188b0e9e0a302f614ff P 698e8d4afb8f46c08c4075b41d506c40 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9be3bc22b32c4eb6bae392ab7519b984 in term 1.
09:03:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.462879 5587 leader_election.cc:304] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9be3bc22b32c4eb6bae392ab7519b984, e9311c63c2b94110958d997d35d1e689; no voters:
09:03:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.463184 5743 raft_consensus.cc:2798] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 1 FOLLOWER]: Leader election won for term 1
09:03:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.463562 5743 raft_consensus.cc:695] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [term 1 LEADER]: Becoming Leader. State: Replica: 9be3bc22b32c4eb6bae392ab7519b984, State: Running, Role: LEADER
09:03:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.464083 5743 consensus_queue.cc:237] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.469174 5176 catalog_manager.cc:5526] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195). New cstate: current_term: 1 leader_uuid: "9be3bc22b32c4eb6bae392ab7519b984" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } health_report { overall_health: HEALTHY } } }
09:03:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.470274 5712 tablet.cc:2367] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:26.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:26.473023 5570 tablet.cc:2367] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:26.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.734742 5745 raft_consensus.cc:491] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:26.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.734961 5745 raft_consensus.cc:513] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.736603 5745 leader_election.cc:290] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309), 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.743404 5377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "198c32aa056e4b82adbcf0e0bd4caac0" candidate_uuid: "698e8d4afb8f46c08c4075b41d506c40" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9311c63c2b94110958d997d35d1e689" is_pre_election: true
09:03:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.743742 5377 raft_consensus.cc:2463] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 698e8d4afb8f46c08c4075b41d506c40 in term 0.
09:03:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.743906 5662 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "198c32aa056e4b82adbcf0e0bd4caac0" candidate_uuid: "698e8d4afb8f46c08c4075b41d506c40" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9be3bc22b32c4eb6bae392ab7519b984" is_pre_election: true
09:03:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.744258 5662 raft_consensus.cc:2463] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 698e8d4afb8f46c08c4075b41d506c40 in term 0.
09:03:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.744254 5444 leader_election.cc:304] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 698e8d4afb8f46c08c4075b41d506c40, e9311c63c2b94110958d997d35d1e689; no voters:
09:03:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.744683 5745 raft_consensus.cc:2798] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.744812 5745 raft_consensus.cc:491] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.744918 5745 raft_consensus.cc:3054] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Advancing to term 1
09:03:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.746825 5745 raft_consensus.cc:513] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.747453 5745 leader_election.cc:290] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [CANDIDATE]: Term 1 election: Requested vote from peers e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309), 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.747833 5377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "198c32aa056e4b82adbcf0e0bd4caac0" candidate_uuid: "698e8d4afb8f46c08c4075b41d506c40" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9311c63c2b94110958d997d35d1e689"
09:03:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.748029 5377 raft_consensus.cc:3054] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Advancing to term 1
09:03:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.747984 5662 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "198c32aa056e4b82adbcf0e0bd4caac0" candidate_uuid: "698e8d4afb8f46c08c4075b41d506c40" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9be3bc22b32c4eb6bae392ab7519b984"
09:03:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.748229 5662 raft_consensus.cc:3054] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Advancing to term 1
09:03:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.749892 5377 raft_consensus.cc:2463] T 198c32aa056e4b82adbcf0e0bd4caac0 P e9311c63c2b94110958d997d35d1e689 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 698e8d4afb8f46c08c4075b41d506c40 in term 1.
09:03:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.750172 5662 raft_consensus.cc:2463] T 198c32aa056e4b82adbcf0e0bd4caac0 P 9be3bc22b32c4eb6bae392ab7519b984 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 698e8d4afb8f46c08c4075b41d506c40 in term 1.
09:03:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.750260 5444 leader_election.cc:304] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 698e8d4afb8f46c08c4075b41d506c40, e9311c63c2b94110958d997d35d1e689; no voters:
09:03:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.750547 5745 raft_consensus.cc:2798] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 1 FOLLOWER]: Leader election won for term 1
09:03:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.750977 5745 raft_consensus.cc:695] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [term 1 LEADER]: Becoming Leader. State: Replica: 698e8d4afb8f46c08c4075b41d506c40, State: Running, Role: LEADER
09:03:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.751543 5745 consensus_queue.cc:237] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.756948 5176 catalog_manager.cc:5526] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 reported cstate change: term changed from 0 to 1, leader changed from <none> to 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194). New cstate: current_term: 1 leader_uuid: "698e8d4afb8f46c08c4075b41d506c40" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } health_report { overall_health: UNKNOWN } } }
09:03:26.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.844735 5743 consensus_queue.cc:1035] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [LEADER]: Connected to new peer: Peer: permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:26.853749 5743 consensus_queue.cc:1035] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.012879 5176 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39690:
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.013867 5176 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:03:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.024972 5356 tablet_service.cc:1467] Processing CreateTablet for tablet 2f3bef2ea7104d5383ac0964f2f87f4d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:03:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.025110 5500 tablet_service.cc:1467] Processing CreateTablet for tablet 2f3bef2ea7104d5383ac0964f2f87f4d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:03:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.025993 5356 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f3bef2ea7104d5383ac0964f2f87f4d. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.026000 5500 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f3bef2ea7104d5383ac0964f2f87f4d. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.031239 5775 tablet_bootstrap.cc:492] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: Bootstrap starting.
09:03:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.031697 5642 tablet_service.cc:1467] Processing CreateTablet for tablet 2f3bef2ea7104d5383ac0964f2f87f4d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:03:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.032672 5642 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f3bef2ea7104d5383ac0964f2f87f4d. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.033142 5775 tablet_bootstrap.cc:654] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: Neither blocks nor log segments found. Creating new log.
09:03:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.036934 5777 tablet_bootstrap.cc:492] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: Bootstrap starting.
09:03:27.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.038601 5777 tablet_bootstrap.cc:654] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: Neither blocks nor log segments found. Creating new log.
09:03:27.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.042331 5780 tablet_bootstrap.cc:492] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: Bootstrap starting.
09:03:27.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.043821 5777 tablet_bootstrap.cc:492] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: No bootstrap required, opened a new log
09:03:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.043898 5775 tablet_bootstrap.cc:492] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: No bootstrap required, opened a new log
09:03:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.043972 5777 ts_tablet_manager.cc:1397] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.006s
09:03:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.044044 5775 ts_tablet_manager.cc:1397] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: Time spent bootstrapping tablet: real 0.013s user 0.008s sys 0.004s
09:03:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.044322 5780 tablet_bootstrap.cc:654] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: Neither blocks nor log segments found. Creating new log.
09:03:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.044708 5777 raft_consensus.cc:357] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.044907 5775 raft_consensus.cc:357] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.045061 5777 raft_consensus.cc:383] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.045161 5777 raft_consensus.cc:738] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 698e8d4afb8f46c08c4075b41d506c40, State: Initialized, Role: FOLLOWER
09:03:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.045310 5775 raft_consensus.cc:383] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.045433 5777 consensus_queue.cc:260] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.045523 5775 raft_consensus.cc:738] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9311c63c2b94110958d997d35d1e689, State: Initialized, Role: FOLLOWER
09:03:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.045850 5775 consensus_queue.cc:260] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.046176 5777 ts_tablet_manager.cc:1428] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
09:03:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.046108 5745 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.046383 5745 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.046908 5742 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.047245 5742 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.049986 5775 ts_tablet_manager.cc:1428] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
09:03:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.053324 5780 tablet_bootstrap.cc:492] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: No bootstrap required, opened a new log
09:03:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.053514 5780 ts_tablet_manager.cc:1397] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.000s
09:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.054289 5780 raft_consensus.cc:357] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.054483 5780 raft_consensus.cc:383] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.054567 5780 raft_consensus.cc:738] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9be3bc22b32c4eb6bae392ab7519b984, State: Initialized, Role: FOLLOWER
09:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.054795 5780 consensus_queue.cc:260] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.055308 5780 ts_tablet_manager.cc:1428] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
09:03:27.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.055287 5743 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.055627 5743 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.216320 5742 raft_consensus.cc:491] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.216542 5742 raft_consensus.cc:513] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.218209 5742 leader_election.cc:290] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447), 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.226202 5662 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2f3bef2ea7104d5383ac0964f2f87f4d" candidate_uuid: "e9311c63c2b94110958d997d35d1e689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9be3bc22b32c4eb6bae392ab7519b984" is_pre_election: true
09:03:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.226459 5662 raft_consensus.cc:2463] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9311c63c2b94110958d997d35d1e689 in term 0.
09:03:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.227097 5299 leader_election.cc:304] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9be3bc22b32c4eb6bae392ab7519b984, e9311c63c2b94110958d997d35d1e689; no voters:
09:03:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.227197 5520 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2f3bef2ea7104d5383ac0964f2f87f4d" candidate_uuid: "e9311c63c2b94110958d997d35d1e689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "698e8d4afb8f46c08c4075b41d506c40" is_pre_election: true
09:03:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.227424 5520 raft_consensus.cc:2463] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9311c63c2b94110958d997d35d1e689 in term 0.
09:03:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.227486 5742 raft_consensus.cc:2798] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.227633 5742 raft_consensus.cc:491] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.227742 5742 raft_consensus.cc:3054] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 0 FOLLOWER]: Advancing to term 1
09:03:27.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.229740 5742 raft_consensus.cc:513] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.230293 5742 leader_election.cc:290] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [CANDIDATE]: Term 1 election: Requested vote from peers 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447), 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.230646 5520 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2f3bef2ea7104d5383ac0964f2f87f4d" candidate_uuid: "e9311c63c2b94110958d997d35d1e689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "698e8d4afb8f46c08c4075b41d506c40"
09:03:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.230790 5662 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2f3bef2ea7104d5383ac0964f2f87f4d" candidate_uuid: "e9311c63c2b94110958d997d35d1e689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9be3bc22b32c4eb6bae392ab7519b984"
09:03:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.230870 5520 raft_consensus.cc:3054] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 0 FOLLOWER]: Advancing to term 1
09:03:27.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.231009 5662 raft_consensus.cc:3054] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 0 FOLLOWER]: Advancing to term 1
09:03:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.233680 5662 raft_consensus.cc:2463] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9311c63c2b94110958d997d35d1e689 in term 1.
09:03:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.233681 5520 raft_consensus.cc:2463] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9311c63c2b94110958d997d35d1e689 in term 1.
09:03:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.234081 5298 leader_election.cc:304] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 698e8d4afb8f46c08c4075b41d506c40, e9311c63c2b94110958d997d35d1e689; no voters:
09:03:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.234344 5742 raft_consensus.cc:2798] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 1 FOLLOWER]: Leader election won for term 1
09:03:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.234759 5742 raft_consensus.cc:695] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [term 1 LEADER]: Becoming Leader. State: Replica: e9311c63c2b94110958d997d35d1e689, State: Running, Role: LEADER
09:03:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.235297 5742 consensus_queue.cc:237] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } }
09:03:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.238091 5785 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: TxnStatusTablet state changed. Reason: New leader e9311c63c2b94110958d997d35d1e689. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.238387 5785 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: This TxnStatusTablet replica's current role is: LEADER
09:03:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.239204 5176 catalog_manager.cc:5526] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 reported cstate change: term changed from 0 to 1, leader changed from <none> to e9311c63c2b94110958d997d35d1e689 (127.4.218.193). New cstate: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } health_report { overall_health: UNKNOWN } } }
09:03:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.240425 5787 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
09:03:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.240690 5787 txn_status_manager.cc:930] Loading transaction status metadata into memory...
09:03:27.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.271898 5745 consensus_queue.cc:1035] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.284804 5745 consensus_queue.cc:1035] T 198c32aa056e4b82adbcf0e0bd4caac0 P 698e8d4afb8f46c08c4075b41d506c40 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.287878 5661 raft_consensus.cc:1270] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984 [term 1 FOLLOWER]: Refusing update from remote peer e9311c63c2b94110958d997d35d1e689: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:03:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.288156 5520 raft_consensus.cc:1270] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40 [term 1 FOLLOWER]: Refusing update from remote peer e9311c63c2b94110958d997d35d1e689: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:03:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.289333 5785 consensus_queue.cc:1035] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.290666 5742 consensus_queue.cc:1035] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689 [LEADER]: Connected to new peer: Peer: permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.300477 5745 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: TxnStatusTablet state changed. Reason: New leader e9311c63c2b94110958d997d35d1e689. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.300783 5745 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.306025 5743 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: TxnStatusTablet state changed. Reason: New leader e9311c63c2b94110958d997d35d1e689. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.306298 5743 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.308024 5760 mvcc.cc:204] Tried to move back new op lower bound from 7113057105041051648 to 7113057104839237632. Current Snapshot: MvccSnapshot[applied={T|T < 7113057105041051648}]
09:03:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.308480 5763 mvcc.cc:204] Tried to move back new op lower bound from 7113057105041051648 to 7113057104839237632. Current Snapshot: MvccSnapshot[applied={T|T < 7113057105041051648}]
09:03:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.308904 5758 mvcc.cc:204] Tried to move back new op lower bound from 7113057105041051648 to 7113057104839237632. Current Snapshot: MvccSnapshot[applied={T|T < 7113057105041051648}]
09:03:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.315855 5743 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.316107 5743 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P 9be3bc22b32c4eb6bae392ab7519b984: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.317097 5745 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.317441 5745 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P 698e8d4afb8f46c08c4075b41d506c40: This TxnStatusTablet replica's current role is: FOLLOWER
09:03:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.322017 5742 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.322299 5742 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: This TxnStatusTablet replica's current role is: LEADER
09:03:27.330 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
09:03:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.330013 5742 tablet_replica.cc:440] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e9311c63c2b94110958d997d35d1e689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e9311c63c2b94110958d997d35d1e689" member_type: VOTER last_known_addr { host: "127.4.218.193" port: 33309 } } peers { permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" member_type: VOTER last_known_addr { host: "127.4.218.194" port: 42447 } } peers { permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" member_type: VOTER last_known_addr { host: "127.4.218.195" port: 39399 } } }
09:03:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.330256 5742 tablet_replica.cc:443] T 2f3bef2ea7104d5383ac0964f2f87f4d P e9311c63c2b94110958d997d35d1e689: This TxnStatusTablet replica's current role is: LEADER
09:03:27.339 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 70f8350d8f6f45cd804021cac5ee71c2
09:03:27.339 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.218.252:35957
09:03:27.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:03:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.341166 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 5143
09:03:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.342361 5217 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:27.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.476310 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 5143
09:03:27.498 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.218.252:35957(127.4.218.252:35957)] handling channelInactive
09:03:27.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state READY due to: connection closed
09:03:27.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:27.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.499866 5709 heartbeater.cc:643] Failed to heartbeat to 127.4.218.252:35957 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.4.218.252:35957 (error 108)
09:03:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.776528 5806 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader f51ede745e8b42f4b48d705dc3953e5a)
09:03:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.776557 5807 raft_consensus.cc:491] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader f51ede745e8b42f4b48d705dc3953e5a)
09:03:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.776744 5806 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.776760 5807 raft_consensus.cc:513] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.777980 5806 leader_election.cc:290] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.778620 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "feb6fc2babbb4eba82e1a977da112d45" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f" is_pre_election: true
09:03:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.778898 5106 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate feb6fc2babbb4eba82e1a977da112d45 in term 1.
09:03:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.779060 5024 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7549c035609942c2a1e7a1a6e1fc884f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "feb6fc2babbb4eba82e1a977da112d45" is_pre_election: true
09:03:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.779310 5024 raft_consensus.cc:2463] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7549c035609942c2a1e7a1a6e1fc884f in term 1.
09:03:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.779444 4999 leader_election.cc:304] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f, feb6fc2babbb4eba82e1a977da112d45; no voters:
09:03:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.780932 5083 leader_election.cc:304] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f, feb6fc2babbb4eba82e1a977da112d45; no voters:
09:03:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.781599 5806 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Leader pre-election won for term 2
09:03:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.781837 5806 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Starting leader election (detected failure of leader f51ede745e8b42f4b48d705dc3953e5a)
09:03:27.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.782058 5806 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 1 FOLLOWER]: Advancing to term 2
09:03:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.783946 5807 leader_election.cc:290] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.784548 5807 raft_consensus.cc:2798] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:03:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.785079 5807 raft_consensus.cc:491] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Starting leader election (detected failure of leader f51ede745e8b42f4b48d705dc3953e5a)
09:03:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.785511 5807 raft_consensus.cc:3054] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 1 FOLLOWER]: Advancing to term 2
09:03:27.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.786022 5806 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:27.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.787894 5024 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7549c035609942c2a1e7a1a6e1fc884f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "feb6fc2babbb4eba82e1a977da112d45"
09:03:27.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.788727 5806 leader_election.cc:290] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 2 election: Requested vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:27.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.790243 5807 raft_consensus.cc:513] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:27.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.790791 5024 raft_consensus.cc:2388] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 7549c035609942c2a1e7a1a6e1fc884f in current term 2: Already voted for candidate feb6fc2babbb4eba82e1a977da112d45 in this term.
09:03:27.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.791392 5082 leader_election.cc:336] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.791924 5082 leader_election.cc:336] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.792158 5082 leader_election.cc:304] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f; no voters: f51ede745e8b42f4b48d705dc3953e5a, feb6fc2babbb4eba82e1a977da112d45
09:03:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.792276 5807 leader_election.cc:290] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 2 election: Requested vote from peers 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.792459 5001 leader_election.cc:336] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.792322 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "feb6fc2babbb4eba82e1a977da112d45" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:27.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.792960 5106 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate feb6fc2babbb4eba82e1a977da112d45 in current term 2: Already voted for candidate 7549c035609942c2a1e7a1a6e1fc884f in this term.
09:03:27.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.793358 5806 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
09:03:27.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.793423 5001 leader_election.cc:336] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:27.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.793624 4999 leader_election.cc:304] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: feb6fc2babbb4eba82e1a977da112d45; no voters: 7549c035609942c2a1e7a1a6e1fc884f, f51ede745e8b42f4b48d705dc3953e5a
09:03:27.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:27.793911 5807 raft_consensus.cc:2743] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
09:03:27.836 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.837 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.4.218.252:35957(127.4.218.252:35957) for tablet Kudu Master: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.855 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.856 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.856 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.857 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:27.875 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.876 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.876 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.876 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.877 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:27.895 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.897 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:27.899401 5566 heartbeater.cc:643] Failed to heartbeat to 127.4.218.252:35957 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:27.900 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:27.915 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.916 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.916 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.917 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.918 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:27.955 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.956 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.957 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.957 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.957 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:27.995 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:27.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.997 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.997 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:27.997 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:27.998 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [1ms] received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [20ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [40ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [41ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [61ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [161ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [161ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, deferred=Deferred@1656692673(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@23d0c5d6 -> (continuation of Deferred@542478878 after retry RPC after error@1011981640) -> (continuation of Deferred@646634516 after retry RPC after error@55661414) -> (continuation of Deferred@261682288 after retry RPC after error@1058247446) -> (continuation of Deferred@1326023324 after retry RPC after error@1179263608) -> (continuation of Deferred@247035317 after retry RPC after error@764533847), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@2e8fb61b -> (continuation of Deferred@542478878 after retry RPC after error@1011981640) -> (continuation of Deferred@646634516 after retry RPC after error@55661414) -> (continuation of Deferred@261682288 after retry RPC after error@1058247446) -> (continuation of Deferred@1326023324 after retry RPC after error@1179263608) -> (continuation of Deferred@247035317 after retry RPC after error@764533847)))
09:03:27.999 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [1ms] received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [20ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [40ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [41ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [61ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [161ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [161ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, deferred=Deferred@1656692673(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@23d0c5d6 -> (continuation of Deferred@542478878 after retry RPC after error@1011981640) -> (continuation of Deferred@646634516 after retry RPC after error@55661414) -> (continuation of Deferred@261682288 after retry RPC after error@1058247446) -> (continuation of Deferred@1326023324 after retry RPC after error@1179263608) -> (continuation of Deferred@247035317 after retry RPC after error@764533847), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@2e8fb61b -> (continuation of Deferred@542478878 after retry RPC after error@1011981640) -> (continuation of Deferred@646634516 after retry RPC after error@55661414) -> (continuation of Deferred@261682288 after retry RPC after error@1058247446) -> (continuation of Deferred@1326023324 after retry RPC after error@1179263608) -> (continuation of Deferred@247035317 after retry RPC after error@764533847)))
09:03:28.015 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.017 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.035 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.036 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.036 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.037 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.038 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.055 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.058 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.075 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.077 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.077 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.077 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.077 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.095 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.096 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.096 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.096 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.097 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.115 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.118 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.173355 5807 raft_consensus.cc:491] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:28.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.173578 5807 raft_consensus.cc:513] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:28.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.174330 5807 leader_election.cc:290] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:28.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.174958 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "feb6fc2babbb4eba82e1a977da112d45" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f" is_pre_election: true
09:03:28.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.175061 5001 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111) [suppressed 10 similar messages]
09:03:28.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.175223 5106 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate feb6fc2babbb4eba82e1a977da112d45 in term 2.
09:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.175988 4999 leader_election.cc:304] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f, feb6fc2babbb4eba82e1a977da112d45; no voters:
09:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.176281 5001 leader_election.cc:336] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.176394 5807 raft_consensus.cc:2798] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Leader pre-election won for term 3
09:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.176497 5807 raft_consensus.cc:491] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.176622 5807 raft_consensus.cc:3054] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 2 FOLLOWER]: Advancing to term 3
09:03:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.178516 5807 raft_consensus.cc:513] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.179196 5807 leader_election.cc:290] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 3 election: Requested vote from peers 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.179653 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "feb6fc2babbb4eba82e1a977da112d45" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.179885 5106 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 2 FOLLOWER]: Advancing to term 3
09:03:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.180713 5001 leader_election.cc:336] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:28.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.182063 5106 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate feb6fc2babbb4eba82e1a977da112d45 in term 3.
09:03:28.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.182519 4999 leader_election.cc:304] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f, feb6fc2babbb4eba82e1a977da112d45; no voters: f51ede745e8b42f4b48d705dc3953e5a
09:03:28.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.182826 5807 raft_consensus.cc:2798] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 3 FOLLOWER]: Leader election won for term 3
09:03:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.183019 5807 raft_consensus.cc:695] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [term 3 LEADER]: Becoming Leader. State: Replica: feb6fc2babbb4eba82e1a977da112d45, State: Running, Role: LEADER
09:03:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.183485 5807 consensus_queue.cc:237] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.185137 5810 sys_catalog.cc:455] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: SysCatalogTable state changed. Reason: New leader feb6fc2babbb4eba82e1a977da112d45. Latest consensus state: current_term: 3 leader_uuid: "feb6fc2babbb4eba82e1a977da112d45" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.185467 5810 sys_catalog.cc:458] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: This master's current role is: LEADER
09:03:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.186829 5819 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:03:28.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.189230 5819 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]
09:03:28.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.189747 5819 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca]
09:03:28.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.191426 5819 tablet_loader.cc:96] loaded metadata for tablet 198c32aa056e4b82adbcf0e0bd4caac0 (table txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2])
09:03:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.191901 5819 tablet_loader.cc:96] loaded metadata for tablet 25c77200f9214188b0e9e0a302f614ff (table txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2])
09:03:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.192234 5819 tablet_loader.cc:96] loaded metadata for tablet 2f3bef2ea7104d5383ac0964f2f87f4d (table kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca])
09:03:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.192535 5819 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:03:28.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.193521 5819 catalog_manager.cc:1260] Loaded cluster ID: 3d8e045736ab427bbdcb73e0d007dcad
09:03:28.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.193624 5819 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:03:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.194813 5819 catalog_manager.cc:1505] Loading token signing keys...
09:03:28.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.195729 5819 catalog_manager.cc:5910] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45: Loaded TSK: 0
09:03:28.196 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.196789 5819 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:03:28.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.198 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:28.199 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [42ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [43ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [103ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [182ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [183ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, deferred=Deferred@1072270264(state=PENDING, result=null, callback=(continuation of Deferred@1605462814 after retry RPC after error@99065919) -> (continuation of Deferred@407323483 after retry RPC after error@1803599776) -> (continuation of Deferred@1069400149 after retry RPC after error@132344261) -> (continuation of Deferred@631025657 after retry RPC after error@1267689813) -> (continuation of Deferred@442611481 after retry RPC after error@114317521) -> (continuation of Deferred@1190386837 after retry RPC after error@1980154079), errback=(continuation of Deferred@1605462814 after retry RPC after error@99065919) -> (continuation of Deferred@407323483 after retry RPC after error@1803599776) -> (continuation of Deferred@1069400149 after retry RPC after error@132344261) -> (continuation of Deferred@631025657 after retry RPC after error@1267689813) -> (continuation of Deferred@442611481 after retry RPC after error@114317521) -> (continuation of Deferred@1190386837 after retry RPC after error@1980154079)))
09:03:28.199 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [42ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [43ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [81ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [103ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [182ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [183ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, deferred=Deferred@1072270264(state=PENDING, result=null, callback=(continuation of Deferred@1605462814 after retry RPC after error@99065919) -> (continuation of Deferred@407323483 after retry RPC after error@1803599776) -> (continuation of Deferred@1069400149 after retry RPC after error@132344261) -> (continuation of Deferred@631025657 after retry RPC after error@1267689813) -> (continuation of Deferred@442611481 after retry RPC after error@114317521) -> (continuation of Deferred@1190386837 after retry RPC after error@1980154079), errback=(continuation of Deferred@1605462814 after retry RPC after error@99065919) -> (continuation of Deferred@407323483 after retry RPC after error@1803599776) -> (continuation of Deferred@1069400149 after retry RPC after error@132344261) -> (continuation of Deferred@631025657 after retry RPC after error@1267689813) -> (continuation of Deferred@442611481 after retry RPC after error@114317521) -> (continuation of Deferred@1190386837 after retry RPC after error@1980154079)))
09:03:28.216 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:28.217 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.217 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.217 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.252:35957: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957
09:03:28.220 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:03:28.221 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=5), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [2ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: Network error: Failed to connect to peer master-127.4.218.252:35957(127.4.218.252:35957): Connection refused: /127.4.218.252:35957, [3ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [5ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: OK, deferred=null) after lookup
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.276005 5014 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60594:
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:03:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.278415 5014 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:03:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.279978 5001 consensus_peers.cc:487] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 -> Peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Couldn't send request to peer f51ede745e8b42f4b48d705dc3953e5a. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:03:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.289815 5106 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Refusing update from remote peer feb6fc2babbb4eba82e1a977da112d45: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
09:03:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.290518 5810 consensus_queue.cc:1035] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
09:03:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.292495 5806 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: New leader feb6fc2babbb4eba82e1a977da112d45. Latest consensus state: current_term: 3 leader_uuid: "feb6fc2babbb4eba82e1a977da112d45" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.292793 5806 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.304628 5807 sys_catalog.cc:455] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "feb6fc2babbb4eba82e1a977da112d45" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.304934 5807 sys_catalog.cc:458] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 [sys.catalog]: This master's current role is: LEADER
09:03:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.304826 5806 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "feb6fc2babbb4eba82e1a977da112d45" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:28.305096 5806 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:28.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.333941 5422 heartbeater.cc:643] Failed to heartbeat to 127.4.218.252:35957 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111)
09:03:28.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:28.738934 5001 consensus_peers.cc:487] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 -> Peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Couldn't send request to peer f51ede745e8b42f4b48d705dc3953e5a. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:03:29.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.229055 5001 consensus_peers.cc:487] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 -> Peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Couldn't send request to peer f51ede745e8b42f4b48d705dc3953e5a. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:03:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.339879 5423 heartbeater.cc:502] Master 127.4.218.254:46469 was elected leader, sending a full tablet report...
09:03:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.344053 4983 master_runner.cc:186] Error getting master registration for 127.4.218.252:35957: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111),
09:03:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.344295 4983 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:03:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.392611 5710 heartbeater.cc:502] Master 127.4.218.254:46469 was elected leader, sending a full tablet report...
09:03:29.505 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 25c77200f9214188b0e9e0a302f614ff for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
09:03:29.506 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 198c32aa056e4b82adbcf0e0bd4caac0 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
09:03:29.506 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=25c77200f9214188b0e9e0a302f614ff@[698e8d4afb8f46c08c4075b41d506c40(127.4.218.194:42447),9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399)[L],e9311c63c2b94110958d997d35d1e689(127.4.218.193:33309)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=198c32aa056e4b82adbcf0e0bd4caac0@[698e8d4afb8f46c08c4075b41d506c40(127.4.218.194:42447)[L],9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399),e9311c63c2b94110958d997d35d1e689(127.4.218.193:33309)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
09:03:29.506 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.218.254:46469, [1ms] Sub RPC GetTableLocations: received response from server master-127.4.218.254:46469: OK, deferred=null) after lookup
09:03:29.506 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399)] connecting to peer
09:03:29.507 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399)] Successfully connected to peer
09:03:29.519 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe4c8b10e, L:/127.0.0.1:50532 - R:/127.4.218.195:39399] using TOKEN/null
09:03:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.523355 5629 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.4.218.252:35957)
09:03:29.558 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.218.252:35957
09:03:29.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.559621 4971 external_mini_cluster.cc:1351] Running /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/logs
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/info.pb
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.252:35957
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.252
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41581
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:03:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:03:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.797173 5001 consensus_peers.cc:487] T 00000000000000000000000000000000 P feb6fc2babbb4eba82e1a977da112d45 -> Peer f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957): Couldn't send request to peer f51ede745e8b42f4b48d705dc3953e5a. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.252:35957: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:03:29.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.892503 5866 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:03:29.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.892858 5866 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:29.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.893007 5866 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:03:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.901240 5866 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:03:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.901216 5567 heartbeater.cc:502] Master 127.4.218.254:46469 was elected leader, sending a full tablet report...
09:03:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.901650 5866 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:03:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.901748 5866 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:03:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.901847 5866 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.901909 5866 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:03:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.901991 5866 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.902041 5866 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35251/jwks.json
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.913097 5866 master_runner.cc:386] Master server non-default flags:
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.218.212:43121
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.218.254:46469,127.4.218.253:40603,127.4.218.252:35957
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.218.252:35957
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35251/jwks.json
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/info.pb
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.218.252
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41581
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/logs
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
09:03:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Jan 2025 08:43:21 UTC on 24a791456cd2
09:03:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4123
09:03:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:03:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.914760 5866 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.916843 5866 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:03:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.927314 5873 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.927995 5874 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:29.928632 5876 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.929020 5866 server_base.cc:1034] running on GCE node
09:03:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.929756 5866 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.931243 5866 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.932441 5866 hybrid_clock.cc:648] HybridClock initialized: now 1736586209932394 us; error 46 us; skew 500 ppm
09:03:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.932767 5866 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:29.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.941581 5866 webserver.cc:458] Webserver started at http://127.4.218.252:41581/ using document root <none> and password file <none>
09:03:29.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.942454 5866 fs_manager.cc:362] Metadata directory not provided
09:03:29.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.942616 5866 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:29.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.950420 5866 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
09:03:29.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.954733 5882 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:29.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.956660 5866 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.000s
09:03:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.956844 5866 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data,/tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51ede745e8b42f4b48d705dc3953e5a"
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-11 09:03:23 on dist-test-slave-cspg"
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.957899 5866 fs_report.cc:389] FS layout report
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/wal
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/data
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.983579 5866 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.984587 5866 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:29.984903 5866 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.007110 5866 rpc_server.cc:307] RPC server started. Bound to: 127.4.218.252:35957
09:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.007149 5945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.218.252:35957 every 8 connection(s)
09:03:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.009135 5866 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktIIAp6/test-tmp/mini-kudu-cluster3317822475060257126/master-2/data/info.pb
09:03:30.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.017238 4971 external_mini_cluster.cc:1413] Started /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu as pid 5866
09:03:30.017 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:30.017 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.218.254:46469
09:03:30.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:03:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.017838 5947 sys_catalog.cc:263] Verifying existing consensus state
09:03:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.018955 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 4983
09:03:30.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.020059 5055 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:30.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.022504 5947 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Bootstrap starting.
09:03:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.046197 5947 log.cc:826] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Log is configured to *not* fsync() on all Append() calls
09:03:30.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.064013 5947 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:03:30.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.064653 5947 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Bootstrap complete.
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.075702 5947 raft_consensus.cc:357] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.077093 5947 raft_consensus.cc:738] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f51ede745e8b42f4b48d705dc3953e5a, State: Initialized, Role: FOLLOWER
09:03:30.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.077944 5947 consensus_queue.cc:260] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:30.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.078850 5951 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.079146 5951 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: FOLLOWER
09:03:30.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.080276 5947 sys_catalog.cc:564] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: configured and running, proceeding with master startup.
09:03:30.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.092443 5962 catalog_manager.cc:1260] Loaded cluster ID: 3d8e045736ab427bbdcb73e0d007dcad
09:03:30.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.092567 5962 catalog_manager.cc:1553] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: loading cluster ID for follower catalog manager: success
09:03:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.096784 5962 catalog_manager.cc:1575] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: acquiring CA information for follower catalog manager: success
09:03:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.099555 5962 catalog_manager.cc:1603] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:30.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.160949 5911 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 1 FOLLOWER]: Advancing to term 3
09:03:30.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.164278 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 4983
09:03:30.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.182219 5968 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "feb6fc2babbb4eba82e1a977da112d45" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.182632 5968 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: FOLLOWER
09:03:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.183287 5951 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: New leader feb6fc2babbb4eba82e1a977da112d45. Latest consensus state: current_term: 3 leader_uuid: "feb6fc2babbb4eba82e1a977da112d45" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.183619 5951 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: FOLLOWER
09:03:30.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.189913 5884 connection.cc:537] server connection from 127.0.0.1:40264 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:03:30.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.190654 5083 connection.cc:537] server connection from 127.0.0.1:50196 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:03:30.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:30.191 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.218.254:46469(127.4.218.254:46469)] handling channelInactive
09:03:30.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state READY due to: connection closed
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.193109 5964 negotiation.cc:337] Failed RPC negotiation. Trace:
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.087838 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.4.218.254:46469 (local address 127.0.0.1:60596)
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.088676 (+ 838us) negotiation.cc:107] Waiting for socket to connect
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.088714 (+ 38us) client_negotiation.cc:174] Beginning negotiation
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.088867 (+ 153us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.191802 (+102935us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":562,"thread_start_us":379,"threads_started":1}
09:03:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.193454 5866 master_runner.cc:186] Error getting master registration for 127.4.218.254:46469: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
09:03:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.198513 5866 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:03:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.346110 5422 heartbeater.cc:346] Connected to a master server at 127.4.218.252:35957
09:03:30.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.349840 5900 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e9311c63c2b94110958d997d35d1e689" instance_seqno: 1736586205131198) as {username='slave'} at 127.4.218.193:34125; Asking this server to re-register.
09:03:30.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.350749 5422 heartbeater.cc:463] Registering TS with master...
09:03:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.352710 5900 ts_manager.cc:194] Registered new tserver with Master: e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309)
09:03:30.375 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.376 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.4.218.254:46469(127.4.218.254:46469) for tablet Kudu Master: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.395 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.252:35957(127.4.218.252:35957)] connecting to peer
09:03:30.396 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.396 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.218.252:35957(127.4.218.252:35957)] Successfully connected to peer
09:03:30.397 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.397 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.397 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.254:46469: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.413 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x77eaa038, L:/127.0.0.1:40272 - R:/127.4.218.252:35957] using TOKEN/null
09:03:30.415 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:30.436 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.437 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.437 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.254:46469: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.438 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:30.456 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.457 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.457 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.457 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.254:46469: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.458 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:30.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.465981 5972 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader feb6fc2babbb4eba82e1a977da112d45)
09:03:30.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.466156 5972 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.466885 5972 leader_election.cc:290] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), f51ede745e8b42f4b48d705dc3953e5a (127.4.218.252:35957)
09:03:30.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.468151 5083 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111) [suppressed 6 similar messages]
09:03:30.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.469432 5083 leader_election.cc:336] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111)
09:03:30.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.474684 5911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7549c035609942c2a1e7a1a6e1fc884f" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "f51ede745e8b42f4b48d705dc3953e5a" is_pre_election: true
09:03:30.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.475719 5082 leader_election.cc:304] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f; no voters: f51ede745e8b42f4b48d705dc3953e5a, feb6fc2babbb4eba82e1a977da112d45
09:03:30.476 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.476042 5972 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
09:03:30.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.254:46469: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.478277 5951 raft_consensus.cc:491] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 3 FOLLOWER]: Starting pre-election (detected failure of leader feb6fc2babbb4eba82e1a977da112d45)
09:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.478533 5951 raft_consensus.cc:513] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.480669 5951 leader_election.cc:290] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603)
09:03:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.481254 5887 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111)
09:03:30.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:30.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.487216 5887 leader_election.cc:336] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111)
09:03:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.491619 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f51ede745e8b42f4b48d705dc3953e5a" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f" is_pre_election: true
09:03:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.491868 5106 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f51ede745e8b42f4b48d705dc3953e5a in term 3.
09:03:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.492512 5883 leader_election.cc:304] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f, f51ede745e8b42f4b48d705dc3953e5a; no voters: feb6fc2babbb4eba82e1a977da112d45
09:03:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.492954 5951 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 3 FOLLOWER]: Leader pre-election won for term 4
09:03:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.493083 5951 raft_consensus.cc:491] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 3 FOLLOWER]: Starting leader election (detected failure of leader feb6fc2babbb4eba82e1a977da112d45)
09:03:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.493182 5951 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 3 FOLLOWER]: Advancing to term 4
09:03:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.496496 5951 raft_consensus.cc:513] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.497332 5951 leader_election.cc:290] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 4 election: Requested vote from peers feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469), 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603)
09:03:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.497891 5106 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f51ede745e8b42f4b48d705dc3953e5a" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "7549c035609942c2a1e7a1a6e1fc884f"
09:03:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.498186 5106 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 3 FOLLOWER]: Advancing to term 4
09:03:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.498711 5887 leader_election.cc:336] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111)
09:03:30.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.500280 5106 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f51ede745e8b42f4b48d705dc3953e5a in term 4.
09:03:30.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.500662 5883 leader_election.cc:304] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7549c035609942c2a1e7a1a6e1fc884f, f51ede745e8b42f4b48d705dc3953e5a; no voters: feb6fc2babbb4eba82e1a977da112d45
09:03:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.500978 5951 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 4 FOLLOWER]: Leader election won for term 4
09:03:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.501137 5951 raft_consensus.cc:695] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [term 4 LEADER]: Becoming Leader. State: Replica: f51ede745e8b42f4b48d705dc3953e5a, State: Running, Role: LEADER
09:03:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.501641 5951 consensus_queue.cc:237] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } }
09:03:30.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.503289 5968 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: New leader f51ede745e8b42f4b48d705dc3953e5a. Latest consensus state: current_term: 4 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.503504 5968 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: LEADER
09:03:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.505125 5979 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:03:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.508725 5979 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2]
09:03:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.509307 5979 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca]
09:03:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.509514 5709 heartbeater.cc:346] Connected to a master server at 127.4.218.252:35957
09:03:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.510769 5900 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9be3bc22b32c4eb6bae392ab7519b984" instance_seqno: 1736586206201702) as {username='slave'} at 127.4.218.195:51897; Asking this server to re-register.
09:03:30.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.511319 5709 heartbeater.cc:463] Registering TS with master...
09:03:30.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.511457 5979 tablet_loader.cc:96] loaded metadata for tablet 198c32aa056e4b82adbcf0e0bd4caac0 (table txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2])
09:03:30.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.512007 5979 tablet_loader.cc:96] loaded metadata for tablet 25c77200f9214188b0e9e0a302f614ff (table txn_manager_fallback_rolling [id=70f8350d8f6f45cd804021cac5ee71c2])
09:03:30.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.512375 5979 tablet_loader.cc:96] loaded metadata for tablet 2f3bef2ea7104d5383ac0964f2f87f4d (table kudu_system.kudu_transactions [id=b048d5e5044a42058aae99cd392ce1ca])
09:03:30.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.512462 5900 ts_manager.cc:194] Registered new tserver with Master: 9be3bc22b32c4eb6bae392ab7519b984 (127.4.218.195:39399)
09:03:30.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.512693 5979 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:03:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.513808 5979 catalog_manager.cc:1260] Loaded cluster ID: 3d8e045736ab427bbdcb73e0d007dcad
09:03:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.513906 5979 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:03:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.515403 5979 catalog_manager.cc:1505] Loading token signing keys...
09:03:30.516 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.517 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.517 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.254:46469: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.516569 5979 catalog_manager.cc:5910] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a: Loaded TSK: 0
09:03:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.518620 5979 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:03:30.520 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603; will retry
09:03:30.556 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.218.254:46469(127.4.218.254:46469)] connecting to peer
09:03:30.557 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.557 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.254:46469(127.4.218.254:46469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.557 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.218.254:46469: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469
09:03:30.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:03:30.559 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] sending RPC to server master-127.4.218.254:46469, [1ms] received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [23ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [106ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [108ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [142ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [182ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, deferred=Deferred@108686155(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@48df26b9 -> (continuation of Deferred@1290638363 after retry RPC after error@1102196665) -> (continuation of Deferred@348595447 after retry RPC after error@622604856) -> (continuation of Deferred@91301836 after retry RPC after error@1441668924) -> (continuation of Deferred@2023979498 after retry RPC after error@1013582132) -> (continuation of Deferred@807994930 after retry RPC after error@362539252), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@7694bf84 -> (continuation of Deferred@1290638363 after retry RPC after error@1102196665) -> (continuation of Deferred@348595447 after retry RPC after error@622604856) -> (continuation of Deferred@91301836 after retry RPC after error@1441668924) -> (continuation of Deferred@2023979498 after retry RPC after error@1013582132) -> (continuation of Deferred@807994930 after retry RPC after error@362539252))) after lookup
09:03:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.581207 5887 consensus_peers.cc:487] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a -> Peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Couldn't send request to peer feb6fc2babbb4eba82e1a977da112d45. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:03:30.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.594521 5900 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40288:
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:03:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.597659 5947 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 2 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
09:03:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.608383 5106 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [term 4 FOLLOWER]: Refusing update from remote peer f51ede745e8b42f4b48d705dc3953e5a: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
09:03:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.609117 5968 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
09:03:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.610785 5972 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f51ede745e8b42f4b48d705dc3953e5a. Latest consensus state: current_term: 4 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.611016 5972 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.612165 5951 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.612389 5951 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: This master's current role is: LEADER
09:03:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.613010 5972 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.613248 5972 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7549c035609942c2a1e7a1a6e1fc884f [sys.catalog]: This master's current role is: FOLLOWER
09:03:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.626295 5931 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40272 (request call id 6) took 66 ms (client timeout 66 ms). Trace:
09:03:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.626446 5931 rpcz_store.cc:269] 0111 09:03:30.559941 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:03:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.559999 (+ 58us) service_pool.cc:225] Handling call
09:03:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.626230 (+ 66231us) inbound_call.cc:173] Queueing success response
09:03:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":425,"init.queue_time_us":73,"thread_start_us":791,"threads_started":4}
09:03:30.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.4.218.254:46469, [1ms] received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [23ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [106ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [108ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [142ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [182ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [184ms] sending RPC to server master-127.4.218.252:35957, [185ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@108686155(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@48df26b9 -> (continuation of Deferred@1290638363 after retry RPC after error@1102196665) -> (continuation of Deferred@348595447 after retry RPC after error@622604856) -> (continuation of Deferred@91301836 after retry RPC after error@1441668924) -> (continuation of Deferred@2023979498 after retry RPC after error@1013582132) -> (continuation of Deferred@807994930 after retry RPC after error@362539252) -> (continuation of Deferred@450571211 after retry RPC@1656469044), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@7694bf84 -> (continuation of Deferred@1290638363 after retry RPC after error@1102196665) -> (continuation of Deferred@348595447 after retry RPC after error@622604856) -> (continuation of Deferred@91301836 after retry RPC after error@1441668924) -> (continuation of Deferred@2023979498 after retry RPC after error@1013582132) -> (continuation of Deferred@807994930 after retry RPC after error@362539252) -> (continuation of Deferred@450571211 after retry RPC@1656469044)))
09:03:30.628 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.4.218.254:46469, [1ms] received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [23ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [106ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [108ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [142ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.4.218.252:35957,127.4.218.254:46469,127.4.218.253:40603) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.252:35957, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.254:46469, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.218.253:40603, [182ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.254:46469: Network error: Failed to connect to peer master-127.4.218.254:46469(127.4.218.254:46469): Connection refused: /127.4.218.254:46469, [183ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.252:35957: OK, [184ms] sending RPC to server master-127.4.218.252:35957, [185ms] Sub RPC ConnectToMaster: received response from server master-127.4.218.253:40603: OK, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@108686155(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@48df26b9 -> (continuation of Deferred@1290638363 after retry RPC after error@1102196665) -> (continuation of Deferred@348595447 after retry RPC after error@622604856) -> (continuation of Deferred@91301836 after retry RPC after error@1441668924) -> (continuation of Deferred@2023979498 after retry RPC after error@1013582132) -> (continuation of Deferred@807994930 after retry RPC after error@362539252) -> (continuation of Deferred@450571211 after retry RPC@1656469044), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@7694bf84 -> (continuation of Deferred@1290638363 after retry RPC after error@1102196665) -> (continuation of Deferred@348595447 after retry RPC after error@622604856) -> (continuation of Deferred@91301836 after retry RPC after error@1441668924) -> (continuation of Deferred@2023979498 after retry RPC after error@1013582132) -> (continuation of Deferred@807994930 after retry RPC after error@362539252) -> (continuation of Deferred@450571211 after retry RPC@1656469044)))
09:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.886404 5931 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40272 (request call id 7) took 250 ms (client timeout 250 ms). Trace:
09:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:30.886528 5931 rpcz_store.cc:269] 0111 09:03:30.636263 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.636360 (+ 97us) service_pool.cc:225] Handling call
09:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.886330 (+249970us) inbound_call.cc:173] Queueing success response
09:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:03:30.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@674053855(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@7b16dcc1, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@38d0351b))
09:03:30.887 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@674053855(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@7b16dcc1, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@38d0351b))
09:03:30.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.910549 5566 heartbeater.cc:346] Connected to a master server at 127.4.218.252:35957
09:03:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.911855 5900 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "698e8d4afb8f46c08c4075b41d506c40" instance_seqno: 1736586205688590) as {username='slave'} at 127.4.218.194:37435; Asking this server to re-register.
09:03:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.912390 5566 heartbeater.cc:463] Registering TS with master...
09:03:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.912627 5566 heartbeater.cc:510] Master 127.4.218.252:35957 requested a full tablet report, sending...
09:03:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:30.913807 5900 ts_manager.cc:194] Registered new tserver with Master: 698e8d4afb8f46c08c4075b41d506c40 (127.4.218.194:42447)
09:03:31.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:31.052042 5304 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.187s from last keepalive heartbeat (effective timeout is 1.000s)
09:03:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.066723 5994 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
09:03:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.076637 5887 consensus_peers.cc:487] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a -> Peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Couldn't send request to peer feb6fc2babbb4eba82e1a977da112d45. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:03:31.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.146543 5931 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40272 (request call id 8) took 250 ms (client timeout 250 ms). Trace:
09:03:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.146704 5931 rpcz_store.cc:269] 0111 09:03:30.896235 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:03:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:30.896331 (+ 96us) service_pool.cc:225] Handling call
09:03:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:31.146478 (+250147us) inbound_call.cc:173] Queueing success response
09:03:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:03:31.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@132464163(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@46ff33c6, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@b421f0c))
09:03:31.147 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@132464163(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@46ff33c6, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@b421f0c))
09:03:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.405937 5931 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40272 (request call id 9) took 250 ms (client timeout 250 ms). Trace:
09:03:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.406102 5931 rpcz_store.cc:269] 0111 09:03:31.155745 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:03:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:31.155813 (+ 68us) service_pool.cc:225] Handling call
09:03:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:31.405848 (+250035us) inbound_call.cc:173] Queueing success response
09:03:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:03:31.406 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@839093138(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@1cb9bfd7, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@2367ce85))
09:03:31.407 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.4.218.252:35957, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@839093138(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/74809451@1cb9bfd7, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1765727077@2367ce85))
09:03:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.583966 5887 consensus_peers.cc:487] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a -> Peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Couldn't send request to peer feb6fc2babbb4eba82e1a977da112d45. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:31.620455 5900 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40294:
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:03:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:03:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:03:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:03:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:03:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.621065 5900 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.634387 5931 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40272 (request call id 10) took 217 ms (client timeout 250 ms). Trace:
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:31.634495 5931 rpcz_store.cc:269] 0111 09:03:31.416497 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:31.416582 (+ 85us) service_pool.cc:225] Handling call
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0111 09:03:31.634333 (+217751us) inbound_call.cc:173] Queueing success response
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:03:31.635 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1736586207 last_transition_timestamp: 1736586211
09:03:32.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:32.103439 5887 consensus_peers.cc:487] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a -> Peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Couldn't send request to peer feb6fc2babbb4eba82e1a977da112d45. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:03:32.196 [DEBUG - main] (Connection.java:575) [peer master-127.4.218.252:35957(127.4.218.252:35957)] disconnecting while in state READY
09:03:32.196 [DEBUG - main] (Connection.java:575) [peer master-127.4.218.254:46469(127.4.218.254:46469)] disconnecting while in state TERMINATED
09:03:32.196 [DEBUG - main] (Connection.java:575) [peer master-127.4.218.253:40603(127.4.218.253:40603)] disconnecting while in state READY
09:03:32.196 [DEBUG - main] (Connection.java:575) [peer 9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399)] disconnecting while in state READY
09:03:32.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:03:32.197 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.218.252:35957(127.4.218.252:35957)] handling channelInactive
09:03:32.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.252:35957(127.4.218.252:35957)] cleaning up while in state READY due to: connection closed
09:03:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.197988 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 5283
09:03:32.198 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.218.253:40603(127.4.218.253:40603)] handling channelInactive
09:03:32.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.218.253:40603(127.4.218.253:40603)] cleaning up while in state READY due to: connection closed
09:03:32.198 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399)] handling channelInactive
09:03:32.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9be3bc22b32c4eb6bae392ab7519b984(127.4.218.195:39399)] cleaning up while in state READY due to: connection closed
09:03:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.205840 5417 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:32.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.327852 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 5283
09:03:32.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.350553 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 5427
09:03:32.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.357127 5560 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.398798 5709 heartbeater.cc:502] Master 127.4.218.252:35957 was elected leader, sending a full tablet report...
09:03:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:32.429934 5587 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.218.193:33309: connect: Connection refused (error 111)
09:03:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:32.431340 5587 consensus_peers.cc:487] T 25c77200f9214188b0e9e0a302f614ff P 9be3bc22b32c4eb6bae392ab7519b984 -> Peer e9311c63c2b94110958d997d35d1e689 (127.4.218.193:33309): Couldn't send request to peer e9311c63c2b94110958d997d35d1e689. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.193:33309: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:03:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.454104 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 5427
09:03:32.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.472116 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 5571
09:03:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.478698 5703 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.578768 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 5571
09:03:32.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.595347 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 5062
09:03:32.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.596549 5136 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:32.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:32.632339 5887 consensus_peers.cc:487] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a -> Peer feb6fc2babbb4eba82e1a977da112d45 (127.4.218.254:46469): Couldn't send request to peer feb6fc2babbb4eba82e1a977da112d45. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.254:46469: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
09:03:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.705363 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 5062
09:03:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.725389 4971 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu pid 5866
09:03:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250111 09:03:32.729200 5883 consensus_peers.cc:487] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a -> Peer 7549c035609942c2a1e7a1a6e1fc884f (127.4.218.253:40603): Couldn't send request to peer 7549c035609942c2a1e7a1a6e1fc884f. Status: Network error: Client connection negotiation failed: client connection to 127.4.218.253:40603: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:03:32.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.731770 5941 generic_service.cc:196] Checking for leaks (request via RPC)
09:03:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.845772 4971 external_mini_cluster.cc:1620] Killing /tmp/dist-test-tasktIIAp6/build/asan/bin/kudu with pid 5866
09:03:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250111 09:03:32.845860 5951 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51ede745e8b42f4b48d705dc3953e5a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "f51ede745e8b42f4b48d705dc3953e5a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "feb6fc2babbb4eba82e1a977da112d45" member_type: VOTER last_known_addr { host: "127.4.218.254" port: 46469 } } peers { permanent_uuid: "7549c035609942c2a1e7a1a6e1fc884f" member_type: VOTER last_known_addr { host: "127.4.218.253" port: 40603 } } peers { permanent_uuid: "f51ede745e8b42f4b48d705dc3953e5a" member_type: VOTER last_known_addr { host: "127.4.218.252" port: 35957 } } }
09:03:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-11T09:03:32Z chronyd exiting
09:03:32.997 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)