10:05:28.697 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
10:05:28.698 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkcvsP0/test-tmp
10:05:28.698 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
10:05:28.698 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkcvsP0/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
10:05:28.699 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
10:05:28.989 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:05:28.989 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:05:28.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:28.990964 10582 mini_oidc.cc:150] Starting JWKS server
10:05:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:28.993250 10582 webserver.cc:458] Webserver started at http://127.0.0.1:45569/ using document root <none> and password file <none>
10:05:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:28.993453 10582 mini_oidc.cc:186] Starting OIDC Discovery server
10:05:28.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:28.993937 10582 webserver.cc:458] Webserver started at http://127.0.0.1:37439/ using document root <none> and password file <none>
10:05:29.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:05:29Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
10:05:29.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:05:29Z Disabled control of system clock
10:05:29.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.021809 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/logs
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/info.pb
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.190:43483
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.190
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:05:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:05:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.304626 10593 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.304847 10593 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.304921 10593 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.311802 10593 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:05:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.311887 10593 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:05:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.311923 10593 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.311971 10593 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:05:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.312007 10593 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:05:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.312055 10593 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:05:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.312106 10593 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45569/jwks.json
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.320327 10593 master_runner.cc:386] Master server non-default flags:
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.190:43483
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/info.pb
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.190
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/logs
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.321483 10593 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.322921 10593 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:29.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.332249 10598 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:29.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.332795 10599 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.333648 10601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.334338 10593 server_base.cc:1034] running on GCE node
10:05:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.334967 10593 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.336205 10593 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.337385 10593 hybrid_clock.cc:648] HybridClock initialized: now 1736935529337343 us; error 36 us; skew 500 ppm
10:05:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.337666 10593 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.339573 10593 webserver.cc:458] Webserver started at http://127.10.85.190:39191/ using document root <none> and password file <none>
10:05:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.340245 10593 fs_manager.cc:362] Metadata directory not provided
10:05:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.340332 10593 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.340646 10593 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:05:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.342890 10593 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/instance:
10:05:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d989751ff71c4a4b81b2f08770595162"
10:05:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.343631 10593 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal/instance:
10:05:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d989751ff71c4a4b81b2f08770595162"
10:05:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.347858 10593 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
10:05:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.350639 10607 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.351621 10593 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
10:05:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.351799 10593 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d989751ff71c4a4b81b2f08770595162"
10:05:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.351941 10593 fs_report.cc:389] FS layout report
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/data
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.371385 10593 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.372021 10593 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.372231 10593 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:29.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.391074 10593 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.190:43483
10:05:29.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.391111 10668 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.190:43483 every 8 connection(s)
10:05:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.392186 10593 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/info.pb
10:05:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.394534 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 10593
10:05:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.394731 10582 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal/instance
10:05:29.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.395268 10669 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.396649 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/logs
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data/info.pb
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.189:33575
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.189
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:05:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:05:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.400846 10669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } has no permanent_uuid. Determining permanent_uuid...
10:05:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.414150 10669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } has no permanent_uuid. Determining permanent_uuid...
10:05:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.415026 10608 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.416323 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.189:33575: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.465646 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } attempt: 1
10:05:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.467537 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.189:33575: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.535849 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } attempt: 2
10:05:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.537671 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.189:33575: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.628906 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } attempt: 3
10:05:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.630651 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.189:33575: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:29.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.686877 10671 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:29.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.687153 10671 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:29.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.687309 10671 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.693856 10671 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.693950 10671 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.694000 10671 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.694056 10671 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.694105 10671 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.694187 10671 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:05:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.694247 10671 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45569/jwks.json
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.702313 10671 master_runner.cc:386] Master server non-default flags:
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.189:33575
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data/info.pb
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.189
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/logs
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:29.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.703944 10671 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:29.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.705358 10671 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.716683 10679 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.717370 10680 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.717923 10682 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:29.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.718799 10671 server_base.cc:1034] running on GCE node
10:05:29.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.719360 10671 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:29.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.720450 10671 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.721637 10671 hybrid_clock.cc:648] HybridClock initialized: now 1736935529721598 us; error 31 us; skew 500 ppm
10:05:29.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.721940 10671 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:29.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.728689 10671 webserver.cc:458] Webserver started at http://127.10.85.189:42565/ using document root <none> and password file <none>
10:05:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.729292 10671 fs_manager.cc:362] Metadata directory not provided
10:05:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.729385 10671 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.729661 10671 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:05:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.731835 10671 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data/instance:
10:05:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "585db50113304f22ac16c7e112fcddab"
10:05:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.732549 10671 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal/instance:
10:05:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "585db50113304f22ac16c7e112fcddab"
10:05:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:29.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.736718 10671 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
10:05:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.739600 10688 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.740689 10671 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.740820 10671 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "585db50113304f22ac16c7e112fcddab"
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.741011 10671 fs_report.cc:389] FS layout report
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data/data
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.752974 10671 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.753573 10671 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.753799 10671 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:29.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.771519 10671 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.189:33575
10:05:29.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.771538 10749 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.189:33575 every 8 connection(s)
10:05:29.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.772645 10671 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/data/info.pb
10:05:29.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.773859 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } attempt: 4
10:05:29.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.776425 10750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.780730 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 10671
10:05:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.780898 10582 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-1/wal/instance
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.782004 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/logs
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/info.pb
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.188:35823
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.188
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:05:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.783094 10750 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } has no permanent_uuid. Determining permanent_uuid...
10:05:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.786520 10669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } has no permanent_uuid. Determining permanent_uuid...
10:05:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.788311 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.792136 10750 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } has no permanent_uuid. Determining permanent_uuid...
10:05:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.796844 10750 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } has no permanent_uuid. Determining permanent_uuid...
10:05:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.797731 10691 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:29.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.799131 10750 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.847556 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 1
10:05:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.848400 10750 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 1
10:05:29.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.849448 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:29.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.850185 10750 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.916688 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 2
10:05:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:29.918399 10750 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 2
10:05:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.918661 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:29.919975 10750 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:30.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.011169 10750 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 3
10:05:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.013046 10750 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.018904 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 3
10:05:30.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.020475 10669 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.080797 10754 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.081064 10754 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.081178 10754 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088222 10754 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088302 10754 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088338 10754 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088382 10754 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088418 10754 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088469 10754 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:05:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.088506 10754 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45569/jwks.json
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.096262 10754 master_runner.cc:386] Master server non-default flags:
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.188:35823
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/info.pb
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.188
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/logs
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.097612 10754 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.099157 10754 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:30.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.108493 10760 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.109154 10763 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.109905 10761 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.110283 10754 server_base.cc:1034] running on GCE node
10:05:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.110991 10754 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:30.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.112197 10754 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.113375 10754 hybrid_clock.cc:648] HybridClock initialized: now 1736935530113334 us; error 38 us; skew 500 ppm
10:05:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.113649 10754 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:30.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.115578 10754 webserver.cc:458] Webserver started at http://127.10.85.188:45959/ using document root <none> and password file <none>
10:05:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.116187 10754 fs_manager.cc:362] Metadata directory not provided
10:05:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.116271 10754 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.116520 10754 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:05:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.119057 10754 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/instance:
10:05:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0"
10:05:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:30 on dist-test-slave-jkng"
10:05:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.119828 10754 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal/instance:
10:05:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0"
10:05:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:30 on dist-test-slave-jkng"
10:05:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.124223 10754 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.000s
10:05:30.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.127043 10769 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.128182 10754 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.128312 10754 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0"
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:30 on dist-test-slave-jkng"
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.128471 10754 fs_report.cc:389] FS layout report
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/data
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.155238 10754 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.155925 10754 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.156193 10754 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.156257 10750 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 4
10:05:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.158128 10750 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.175628 10754 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.188:35823
10:05:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.175688 10830 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.188:35823 every 8 connection(s)
10:05:30.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.176826 10754 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/info.pb
10:05:30.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.177803 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 10754
10:05:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.177976 10582 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal/instance
10:05:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.180977 10831 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.187057 10831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } has no permanent_uuid. Determining permanent_uuid...
10:05:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.190728 10669 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 4
10:05:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.201622 10831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } has no permanent_uuid. Determining permanent_uuid...
10:05:30.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.206496 10669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Bootstrap starting.
10:05:30.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.209117 10831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } has no permanent_uuid. Determining permanent_uuid...
10:05:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.210171 10669 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Neither blocks nor log segments found. Creating new log.
10:05:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.211191 10669 log.cc:826] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Log is configured to *not* fsync() on all Append() calls
10:05:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.214002 10669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: No bootstrap required, opened a new log
10:05:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.217176 10831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Bootstrap starting.
10:05:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.220310 10831 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Neither blocks nor log segments found. Creating new log.
10:05:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.220238 10669 raft_consensus.cc:357] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.220438 10669 raft_consensus.cc:383] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.220513 10669 raft_consensus.cc:738] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d989751ff71c4a4b81b2f08770595162, State: Initialized, Role: FOLLOWER
10:05:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.220934 10669 consensus_queue.cc:260] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.221349 10831 log.cc:826] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Log is configured to *not* fsync() on all Append() calls
10:05:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.221697 10839 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.222138 10839 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.222875 10669 sys_catalog.cc:564] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: configured and running, proceeding with master startup.
10:05:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.223917 10831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: No bootstrap required, opened a new log
10:05:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.232483 10831 raft_consensus.cc:357] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.233000 10831 raft_consensus.cc:383] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.233204 10831 raft_consensus.cc:738] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f4e788cbdfb84919a6ecb74d02dbd0b0, State: Initialized, Role: FOLLOWER
10:05:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.233963 10831 consensus_queue.cc:260] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.234498 10851 catalog_manager.cc:1559] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:05:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.234704 10851 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:05:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.235368 10853 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.235812 10853 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.236379 10831 sys_catalog.cc:564] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: configured and running, proceeding with master startup.
10:05:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.239197 10839 raft_consensus.cc:491] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:05:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.239415 10839 raft_consensus.cc:513] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.240885 10839 leader_election.cc:290] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:30.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.241767 10796 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d989751ff71c4a4b81b2f08770595162" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" is_pre_election: true
10:05:30.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.242285 10796 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d989751ff71c4a4b81b2f08770595162 in term 0.
10:05:30.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.243207 10610 leader_election.cc:304] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d989751ff71c4a4b81b2f08770595162, f4e788cbdfb84919a6ecb74d02dbd0b0; no voters:
10:05:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.243727 10839 raft_consensus.cc:2798] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:05:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.243860 10839 raft_consensus.cc:491] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:05:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.243955 10839 raft_consensus.cc:3054] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 0 FOLLOWER]: Advancing to term 1
10:05:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.233044 10593 master_runner.cc:186] Error getting master registration for 127.10.85.189:33575: OK, instance_id { permanent_uuid: "585db50113304f22ac16c7e112fcddab" instance_seqno: 1736935529767084 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:05:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.246326 10593 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:05:30.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.247202 10839 raft_consensus.cc:513] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.248234 10796 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d989751ff71c4a4b81b2f08770595162" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0"
10:05:30.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.248418 10796 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 0 FOLLOWER]: Advancing to term 1
10:05:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.251242 10796 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d989751ff71c4a4b81b2f08770595162 in term 1.
10:05:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.251715 10610 leader_election.cc:304] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d989751ff71c4a4b81b2f08770595162, f4e788cbdfb84919a6ecb74d02dbd0b0; no voters:
10:05:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.253876 10839 leader_election.cc:290] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 1 election: Requested vote from peers 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.254091 10839 raft_consensus.cc:2798] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Leader election won for term 1
10:05:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.254614 10839 raft_consensus.cc:695] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 LEADER]: Becoming Leader. State: Replica: d989751ff71c4a4b81b2f08770595162, State: Running, Role: LEADER
10:05:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.255059 10839 consensus_queue.cc:237] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.259963 10839 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d989751ff71c4a4b81b2f08770595162. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.260226 10839 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: LEADER
10:05:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.261046 10859 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:05:30.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.263738 10859 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:05:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.275331 10796 raft_consensus.cc:1270] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Refusing update from remote peer d989751ff71c4a4b81b2f08770595162: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:05:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.276238 10839 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:30.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.279932 10754 master_runner.cc:186] Error getting master registration for 127.10.85.189:33575: OK, instance_id { permanent_uuid: "585db50113304f22ac16c7e112fcddab" instance_seqno: 1736935529767084 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:05:30.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.312543 10754 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:05:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.313560 10853 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d989751ff71c4a4b81b2f08770595162. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.313796 10853 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.320416 10867 mvcc.cc:204] Tried to move back new op lower bound from 7114487931987386368 to 7114487931928875008. Current Snapshot: MvccSnapshot[applied={T|T < 7114487931987386368}]
10:05:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.320411 10868 catalog_manager.cc:1559] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:05:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.320683 10868 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:05:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.335996 10857 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.336325 10857 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: LEADER
10:05:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.337935 10853 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.338122 10853 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.344553 10859 catalog_manager.cc:1348] Generated new cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.344661 10859 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:05:30.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.385383 10859 catalog_manager.cc:1371] Generated new certificate authority record
10:05:30.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.389909 10859 catalog_manager.cc:1505] Loading token signing keys...
10:05:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.419867 10859 catalog_manager.cc:5899] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Generated new TSK 0
10:05:30.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.422233 10859 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.450259 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/logs
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data/info.pb
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.129:0
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.85.129
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.129
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:05:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:05:30.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.457307 10750 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } attempt: 5
10:05:30.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.469820 10750 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: Bootstrap starting.
10:05:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.473520 10750 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: Neither blocks nor log segments found. Creating new log.
10:05:30.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.474715 10750 log.cc:826] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: Log is configured to *not* fsync() on all Append() calls
10:05:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.493196 10750 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: No bootstrap required, opened a new log
10:05:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.501677 10750 raft_consensus.cc:357] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.505012 10750 raft_consensus.cc:383] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.505219 10750 raft_consensus.cc:738] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 585db50113304f22ac16c7e112fcddab, State: Initialized, Role: FOLLOWER
10:05:30.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.506016 10750 consensus_queue.cc:260] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.531318 10876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.533686 10876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.534530 10750 sys_catalog.cc:564] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: configured and running, proceeding with master startup.
10:05:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.534607 10714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d989751ff71c4a4b81b2f08770595162" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "585db50113304f22ac16c7e112fcddab"
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.535383 10715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d989751ff71c4a4b81b2f08770595162" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "585db50113304f22ac16c7e112fcddab" is_pre_election: true
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.533502 10877 raft_consensus.cc:491] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.535889 10715 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:48578 (request call id 2) took 293 ms (client timeout 376 ms). Trace:
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.535815 10877 raft_consensus.cc:513] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.536048 10715 rpcz_store.cc:269] 0115 10:05:30.241941 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:30.241994 (+ 53us) service_pool.cc:225] Handling call
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:30.535841 (+293847us) inbound_call.cc:173] Queueing success response
10:05:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:05:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.537787 10634 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "585db50113304f22ac16c7e112fcddab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d989751ff71c4a4b81b2f08770595162" is_pre_election: true
10:05:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.538779 10714 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 0 FOLLOWER]: Advancing to term 1
10:05:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.539405 10877 leader_election.cc:290] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.542320 10714 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d989751ff71c4a4b81b2f08770595162 in term 1.
10:05:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.542939 10796 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "585db50113304f22ac16c7e112fcddab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" is_pre_election: true
10:05:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.543569 10691 leader_election.cc:304] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 585db50113304f22ac16c7e112fcddab; no voters: d989751ff71c4a4b81b2f08770595162, f4e788cbdfb84919a6ecb74d02dbd0b0
10:05:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.544255 10877 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:05:30.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.544816 10713 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Refusing update from remote peer d989751ff71c4a4b81b2f08770595162: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.545948 10714 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:48578 (request call id 3) took 297 ms (client timeout 351 ms). Trace:
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.545887 10857 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [LEADER]: Connected to new peer: Peer: permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.546154 10714 rpcz_store.cc:269] 0115 10:05:30.248002 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:30.248055 (+ 53us) service_pool.cc:225] Handling call
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:30.544316 (+296261us) spinlock_profiling.cc:243] Waited 3.67 ms on lock 0x616000051a54. stack: 00007fc1a4f6f8a6 00007fc1a4f6f660 00007fc1a3f5448d 000000000096d544 000000000096d50d 000000000096ceda 00007fc1b0801b85 00007fc1b44fed94 00007fc1aebd19b6 00007fc1aebd16e5 00007fc1abcd530e 00007fc1abcd41ee 00007fc1abcda425 00007fc1abcdad4f 00007fc1abcdab78 0000000000a71ecc
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:30.545918 (+ 1602us) inbound_call.cc:173] Queueing success response
10:05:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":8431872}
10:05:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.570636 10879 mvcc.cc:204] Tried to move back new op lower bound from 7114487931987386368 to 7114487931928875008. Current Snapshot: MvccSnapshot[applied={T|T < 7114487931987386368}]
10:05:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.581166 10876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.581492 10876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.582070 10877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: New leader d989751ff71c4a4b81b2f08770595162. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.582334 10877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: FOLLOWER
10:05:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.617542 10892 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:05:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.651052 10857 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d989751ff71c4a4b81b2f08770595162" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.651407 10857 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: LEADER
10:05:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.982251 10874 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.982501 10874 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.982596 10874 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:30.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.989799 10874 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.989928 10874 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:05:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.989964 10874 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:05:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.990008 10874 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:05:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:30.990092 10874 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.85.129
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:30.998739 10874 tablet_server_runner.cc:78] Tablet server non-default flags:
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.129:0
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data/info.pb
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.129
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.85.129
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/logs
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.000180 10874 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:31.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.001708 10874 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.012705 10898 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.013442 10899 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.013659 10901 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.014252 10874 server_base.cc:1034] running on GCE node
10:05:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.014712 10874 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:31.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.015487 10874 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:31.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.016681 10874 hybrid_clock.cc:648] HybridClock initialized: now 1736935531016636 us; error 36 us; skew 500 ppm
10:05:31.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.016990 10874 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.018987 10874 webserver.cc:458] Webserver started at http://127.10.85.129:35859/ using document root <none> and password file <none>
10:05:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.019629 10874 fs_manager.cc:362] Metadata directory not provided
10:05:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.019747 10874 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:31.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.020066 10874 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:05:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.022428 10874 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data/instance:
10:05:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a5806654c4148fbb6e787f4ad65533a"
10:05:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.023162 10874 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal/instance:
10:05:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a5806654c4148fbb6e787f4ad65533a"
10:05:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.027762 10874 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.007s sys 0.000s
10:05:31.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.030875 10907 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.032066 10874 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.032225 10874 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a5806654c4148fbb6e787f4ad65533a"
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.032405 10874 fs_report.cc:389] FS layout report
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data/data
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.047770 10874 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.048496 10874 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.048768 10874 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.051440 10874 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:05:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.051548 10874 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:05:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.051672 10874 ts_tablet_manager.cc:610] Registered 0 tablets
10:05:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.051750 10874 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
10:05:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.105713 10874 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.129:43803
10:05:31.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.105731 11031 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.129:43803 every 8 connection(s)
10:05:31.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.106928 10874 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/data/info.pb
10:05:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.109211 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 10874
10:05:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.109468 10582 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-0/wal/instance
10:05:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.109825 11033 heartbeater.cc:346] Connected to a master server at 127.10.85.190:43483
10:05:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.110088 11033 heartbeater.cc:463] Registering TS with master...
10:05:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.111081 11033 heartbeater.cc:509] Master 127.10.85.190:43483 requested a full tablet report, sending...
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.114423 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/logs
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data/info.pb
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.130:0
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.85.130
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.130
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.115084 10623 ts_manager.cc:194] Registered new tserver with Master: 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803)
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.117274 10623 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.85.129:34261
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.118469 11032 heartbeater.cc:346] Connected to a master server at 127.10.85.188:35823
10:05:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.118578 11032 heartbeater.cc:463] Registering TS with master...
10:05:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.119027 11032 heartbeater.cc:509] Master 127.10.85.188:35823 requested a full tablet report, sending...
10:05:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.119441 11034 heartbeater.cc:346] Connected to a master server at 127.10.85.189:33575
10:05:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.119572 11034 heartbeater.cc:463] Registering TS with master...
10:05:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.120124 11034 heartbeater.cc:509] Master 127.10.85.189:33575 requested a full tablet report, sending...
10:05:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.120596 10786 ts_manager.cc:194] Registered new tserver with Master: 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803)
10:05:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.121548 10704 ts_manager.cc:194] Registered new tserver with Master: 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803)
10:05:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.322506 10868 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.322595 10868 catalog_manager.cc:1553] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: loading cluster ID for follower catalog manager: success
10:05:31.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.324940 10868 catalog_manager.cc:1575] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: acquiring CA information for follower catalog manager: success
10:05:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.326349 10868 catalog_manager.cc:1603] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:05:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.423869 11037 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.424121 11037 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.424242 11037 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.430876 11037 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.431018 11037 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:05:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.431082 11037 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:05:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.431140 11037 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:05:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.431294 11037 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.85.130
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.439894 11037 tablet_server_runner.cc:78] Tablet server non-default flags:
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.130:0
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data/info.pb
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.130
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.85.130
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/logs
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.441350 11037 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.442798 11037 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:31.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.453179 11042 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.453706 11043 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.454180 11045 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.455097 11037 server_base.cc:1034] running on GCE node
10:05:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.455523 11037 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:31.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.456368 11037 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:31.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.457588 11037 hybrid_clock.cc:648] HybridClock initialized: now 1736935531457542 us; error 40 us; skew 500 ppm
10:05:31.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.457836 11037 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:31.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.459645 11037 webserver.cc:458] Webserver started at http://127.10.85.130:33259/ using document root <none> and password file <none>
10:05:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.460227 11037 fs_manager.cc:362] Metadata directory not provided
10:05:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.460304 11037 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.460556 11037 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:05:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.462755 11037 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data/instance:
10:05:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34c6f1b7300a46919cb1515f6f721cbc"
10:05:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.463418 11037 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal/instance:
10:05:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34c6f1b7300a46919cb1515f6f721cbc"
10:05:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.467795 11037 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
10:05:31.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.470597 11051 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.471640 11037 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
10:05:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.471817 11037 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34c6f1b7300a46919cb1515f6f721cbc"
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.471961 11037 fs_report.cc:389] FS layout report
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data/data
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.509083 11037 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.509814 11037 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.510025 11037 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.512789 11037 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:05:31.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.512889 11037 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:05:31.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.512989 11037 ts_tablet_manager.cc:610] Registered 0 tablets
10:05:31.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.513027 11037 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:05:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.565532 11037 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.130:45325
10:05:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.565579 11171 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.130:45325 every 8 connection(s)
10:05:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.566704 11037 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/data/info.pb
10:05:31.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.569382 11173 heartbeater.cc:346] Connected to a master server at 127.10.85.190:43483
10:05:31.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.569702 11173 heartbeater.cc:463] Registering TS with master...
10:05:31.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.570549 11173 heartbeater.cc:509] Master 127.10.85.190:43483 requested a full tablet report, sending...
10:05:31.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.571785 10623 ts_manager.cc:194] Registered new tserver with Master: 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.572715 10623 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.85.130:48627
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.574753 11172 heartbeater.cc:346] Connected to a master server at 127.10.85.188:35823
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.574760 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 11037
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.574862 11172 heartbeater.cc:463] Registering TS with master...
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.574903 10582 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-1/wal/instance
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.575184 11172 heartbeater.cc:509] Master 127.10.85.188:35823 requested a full tablet report, sending...
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.575695 11174 heartbeater.cc:346] Connected to a master server at 127.10.85.189:33575
10:05:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.575834 11174 heartbeater.cc:463] Registering TS with master...
10:05:31.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.576124 11174 heartbeater.cc:509] Master 127.10.85.189:33575 requested a full tablet report, sending...
10:05:31.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.576287 10786 ts_manager.cc:194] Registered new tserver with Master: 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:31.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.577505 10704 ts_manager.cc:194] Registered new tserver with Master: 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.578308 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/logs
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data/info.pb
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.131:0
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.85.131
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.131
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:05:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:05:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.625020 10892 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.625131 10892 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: loading cluster ID for follower catalog manager: success
10:05:31.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.628929 10892 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: acquiring CA information for follower catalog manager: success
10:05:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.630986 10892 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.878382 11177 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.878628 11177 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.878715 11177 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.885325 11177 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.885435 11177 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:05:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.885468 11177 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:05:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.885510 11177 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:05:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.885587 11177 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.85.131
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.893795 11177 tablet_server_runner.cc:78] Tablet server non-default flags:
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.131:0
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data/info.pb
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.131
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.85.131
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/logs
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:31.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:31.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:31.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:31.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:31.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.895020 11177 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.896413 11177 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.908771 11184 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.909368 11186 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:31.909946 11183 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:31.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.910636 11177 server_base.cc:1034] running on GCE node
10:05:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.911145 11177 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.912034 11177 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.913244 11177 hybrid_clock.cc:648] HybridClock initialized: now 1736935531913199 us; error 40 us; skew 500 ppm
10:05:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.913471 11177 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.915508 11177 webserver.cc:458] Webserver started at http://127.10.85.131:45763/ using document root <none> and password file <none>
10:05:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.916133 11177 fs_manager.cc:362] Metadata directory not provided
10:05:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.916224 11177 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.916486 11177 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:05:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.918705 11177 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data/instance:
10:05:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb958ffb8ba54460a13472fe93aba131"
10:05:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.919356 11177 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal/instance:
10:05:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb958ffb8ba54460a13472fe93aba131"
10:05:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.923977 11177 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.007s sys 0.001s
10:05:31.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.927129 11192 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.928364 11177 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.928499 11177 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb958ffb8ba54460a13472fe93aba131"
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:31 on dist-test-slave-jkng"
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.928699 11177 fs_report.cc:389] FS layout report
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal
10:05:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data/data
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.946048 11177 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.946717 11177 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.947019 11177 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.949679 11177 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:05:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.949770 11177 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:05:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.949999 11177 ts_tablet_manager.cc:610] Registered 0 tablets
10:05:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.950173 11177 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:05:31.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.998674 11177 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.131:32801
10:05:31.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.998706 11313 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.131:32801 every 8 connection(s)
10:05:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:31.999927 11177 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/data/info.pb
10:05:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.003894 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 11177
10:05:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.004066 10582 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/ts-2/wal/instance
10:05:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.005255 11315 heartbeater.cc:346] Connected to a master server at 127.10.85.190:43483
10:05:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.005604 11315 heartbeater.cc:463] Registering TS with master...
10:05:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.006611 11315 heartbeater.cc:509] Master 127.10.85.190:43483 requested a full tablet report, sending...
10:05:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.008140 10623 ts_manager.cc:194] Registered new tserver with Master: cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.009214 10623 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.85.131:45425
10:05:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.011353 11314 heartbeater.cc:346] Connected to a master server at 127.10.85.188:35823
10:05:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.011483 11314 heartbeater.cc:463] Registering TS with master...
10:05:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.011680 11316 heartbeater.cc:346] Connected to a master server at 127.10.85.189:33575
10:05:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.011818 11316 heartbeater.cc:463] Registering TS with master...
10:05:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.011948 11314 heartbeater.cc:509] Master 127.10.85.188:35823 requested a full tablet report, sending...
10:05:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.012097 11316 heartbeater.cc:509] Master 127.10.85.189:33575 requested a full tablet report, sending...
10:05:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.012984 10786 ts_manager.cc:194] Registered new tserver with Master: cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.013521 10704 ts_manager.cc:194] Registered new tserver with Master: cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.025555 10582 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
10:05:32.027 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:05:32.027 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
10:05:32.028 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.85.190"
port: 43483
}
bound_http_address {
host: "127.10.85.190"
port: 39191
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.85.189"
port: 33575
}
bound_http_address {
host: "127.10.85.189"
port: 42565
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.85.188"
port: 35823
}
bound_http_address {
host: "127.10.85.188"
port: 45959
}
}
}
10:05:32.028 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:05:32.029 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.85.129"
port: 43803
}
bound_http_address {
host: "127.10.85.129"
port: 35859
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.85.130"
port: 45325
}
bound_http_address {
host: "127.10.85.130"
port: 33259
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.85.131"
port: 32801
}
bound_http_address {
host: "127.10.85.131"
port: 45763
}
}
}
10:05:32.029 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
10:05:32.030 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
10:05:32.032 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:32.034 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.85.190:43483(127.10.85.190:43483)] connecting to peer
10:05:32.034 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.85.189:33575(127.10.85.189:33575)] Successfully connected to peer
10:05:32.035 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.85.188:35823(127.10.85.188:35823)] connecting to peer
10:05:32.036 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.85.189
10:05:32.036 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.85.190:43483(127.10.85.190:43483)] Successfully connected to peer
10:05:32.037 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.85.190
10:05:32.048 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x94878c42, L:/127.0.0.1:55644 - R:/127.10.85.190:43483] using SASL/PLAIN
10:05:32.049 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.85.188:35823(127.10.85.188:35823)] Successfully connected to peer
10:05:32.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:05:32.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=19), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [19ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, deferred=null) after lookup
10:05:32.051 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd779dec3, L:/127.0.0.1:48614 - R:/127.10.85.189:33575] using SASL/PLAIN
10:05:32.053 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.85.188
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.053243 10623 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55644:
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
10:05:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
10:05:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
10:05:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.055505 10623 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:05:32.069 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf1b9514a, L:/127.0.0.1:40146 - R:/127.10.85.188:35823] using SASL/PLAIN
10:05:32.079 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dd388c02673449b9d85313eaa39f243
10:05:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.121990 11033 heartbeater.cc:501] Master 127.10.85.190:43483 was elected leader, sending a full tablet report...
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.135542 11246 tablet_service.cc:1467] Processing CreateTablet for tablet 6ae46b0d157a4cd98d06940e64ffe5af (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.135504 10966 tablet_service.cc:1467] Processing CreateTablet for tablet 59ba63bafba443cdb09fba9d4814185f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.136646 10967 tablet_service.cc:1467] Processing CreateTablet for tablet 6ae46b0d157a4cd98d06940e64ffe5af (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.137059 11246 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6ae46b0d157a4cd98d06940e64ffe5af. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.137274 10967 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6ae46b0d157a4cd98d06940e64ffe5af. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.137094 11104 tablet_service.cc:1467] Processing CreateTablet for tablet 59ba63bafba443cdb09fba9d4814185f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:05:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.137022 11105 tablet_service.cc:1467] Processing CreateTablet for tablet 6ae46b0d157a4cd98d06940e64ffe5af (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:05:32.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.137830 11245 tablet_service.cc:1467] Processing CreateTablet for tablet 59ba63bafba443cdb09fba9d4814185f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:05:32.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.138366 10966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 59ba63bafba443cdb09fba9d4814185f. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.138442 11104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 59ba63bafba443cdb09fba9d4814185f. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.139448 11245 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 59ba63bafba443cdb09fba9d4814185f. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.142897 11105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6ae46b0d157a4cd98d06940e64ffe5af. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.156471 11340 tablet_bootstrap.cc:492] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: Bootstrap starting.
10:05:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.156699 11341 tablet_bootstrap.cc:492] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: Bootstrap starting.
10:05:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.158691 11342 tablet_bootstrap.cc:492] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131: Bootstrap starting.
10:05:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.159852 11341 tablet_bootstrap.cc:654] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: Neither blocks nor log segments found. Creating new log.
10:05:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.160290 11340 tablet_bootstrap.cc:654] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: Neither blocks nor log segments found. Creating new log.
10:05:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.161053 11341 log.cc:826] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: Log is configured to *not* fsync() on all Append() calls
10:05:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.161273 11342 tablet_bootstrap.cc:654] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131: Neither blocks nor log segments found. Creating new log.
10:05:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.161373 11340 log.cc:826] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: Log is configured to *not* fsync() on all Append() calls
10:05:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.162457 11342 log.cc:826] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131: Log is configured to *not* fsync() on all Append() calls
10:05:32.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.169643 11341 tablet_bootstrap.cc:492] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: No bootstrap required, opened a new log
10:05:32.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.169868 11342 tablet_bootstrap.cc:492] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131: No bootstrap required, opened a new log
10:05:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.169867 11340 tablet_bootstrap.cc:492] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: No bootstrap required, opened a new log
10:05:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.169880 11341 ts_tablet_manager.cc:1397] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: Time spent bootstrapping tablet: real 0.013s user 0.000s sys 0.011s
10:05:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.170101 11340 ts_tablet_manager.cc:1397] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: Time spent bootstrapping tablet: real 0.014s user 0.007s sys 0.004s
10:05:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.170101 11342 ts_tablet_manager.cc:1397] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.003s
10:05:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.176138 11341 raft_consensus.cc:357] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.176967 11341 raft_consensus.cc:383] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.177160 11341 raft_consensus.cc:738] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34c6f1b7300a46919cb1515f6f721cbc, State: Initialized, Role: FOLLOWER
10:05:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.177876 11341 consensus_queue.cc:260] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.178609 11342 raft_consensus.cc:357] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.178851 11340 raft_consensus.cc:357] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.179185 11340 raft_consensus.cc:383] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.179097 11342 raft_consensus.cc:383] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.179297 11340 raft_consensus.cc:738] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a5806654c4148fbb6e787f4ad65533a, State: Initialized, Role: FOLLOWER
10:05:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.179458 11342 raft_consensus.cc:738] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb958ffb8ba54460a13472fe93aba131, State: Initialized, Role: FOLLOWER
10:05:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.179889 11340 consensus_queue.cc:260] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.180447 11342 consensus_queue.cc:260] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.188083 11340 ts_tablet_manager.cc:1428] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: Time spent starting tablet: real 0.018s user 0.014s sys 0.004s
10:05:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.188622 11342 ts_tablet_manager.cc:1428] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131: Time spent starting tablet: real 0.018s user 0.016s sys 0.002s
10:05:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.190623 11340 tablet_bootstrap.cc:492] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a: Bootstrap starting.
10:05:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.192616 11340 tablet_bootstrap.cc:654] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a: Neither blocks nor log segments found. Creating new log.
10:05:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.196204 11315 heartbeater.cc:501] Master 127.10.85.190:43483 was elected leader, sending a full tablet report...
10:05:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.198163 11340 tablet_bootstrap.cc:492] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a: No bootstrap required, opened a new log
10:05:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.198390 11173 heartbeater.cc:501] Master 127.10.85.190:43483 was elected leader, sending a full tablet report...
10:05:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.200443 11341 ts_tablet_manager.cc:1428] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: Time spent starting tablet: real 0.030s user 0.010s sys 0.014s
10:05:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.200819 11341 tablet_bootstrap.cc:492] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc: Bootstrap starting.
10:05:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.201582 11340 ts_tablet_manager.cc:1397] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.004s
10:05:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.202448 11340 raft_consensus.cc:357] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.202843 11340 raft_consensus.cc:383] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.202953 11340 raft_consensus.cc:738] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a5806654c4148fbb6e787f4ad65533a, State: Initialized, Role: FOLLOWER
10:05:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.203198 11340 consensus_queue.cc:260] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.203526 11341 tablet_bootstrap.cc:654] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc: Neither blocks nor log segments found. Creating new log.
10:05:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.205155 11340 ts_tablet_manager.cc:1428] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
10:05:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.206624 11342 tablet_bootstrap.cc:492] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131: Bootstrap starting.
10:05:32.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.210729 11342 tablet_bootstrap.cc:654] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131: Neither blocks nor log segments found. Creating new log.
10:05:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.211910 11341 tablet_bootstrap.cc:492] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc: No bootstrap required, opened a new log
10:05:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.212126 11341 ts_tablet_manager.cc:1397] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.004s
10:05:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.212916 11341 raft_consensus.cc:357] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.213114 11342 tablet_bootstrap.cc:492] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131: No bootstrap required, opened a new log
10:05:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.213176 11341 raft_consensus.cc:383] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.213299 11341 raft_consensus.cc:738] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34c6f1b7300a46919cb1515f6f721cbc, State: Initialized, Role: FOLLOWER
10:05:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.213399 11342 ts_tablet_manager.cc:1397] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.003s
10:05:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.213585 11341 consensus_queue.cc:260] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.214154 11341 ts_tablet_manager.cc:1428] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
10:05:32.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.214517 11342 raft_consensus.cc:357] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.214694 11342 raft_consensus.cc:383] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.214774 11342 raft_consensus.cc:738] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb958ffb8ba54460a13472fe93aba131, State: Initialized, Role: FOLLOWER
10:05:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.215437 11342 consensus_queue.cc:260] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.217093 11342 ts_tablet_manager.cc:1428] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
10:05:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.223178 11348 raft_consensus.cc:491] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:05:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.223349 11348 raft_consensus.cc:513] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.224818 11348 leader_election.cc:290] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803), 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.231875 10987 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "59ba63bafba443cdb09fba9d4814185f" candidate_uuid: "cb958ffb8ba54460a13472fe93aba131" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a5806654c4148fbb6e787f4ad65533a" is_pre_election: true
10:05:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.231876 11127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "59ba63bafba443cdb09fba9d4814185f" candidate_uuid: "cb958ffb8ba54460a13472fe93aba131" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34c6f1b7300a46919cb1515f6f721cbc" is_pre_election: true
10:05:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.232352 10987 raft_consensus.cc:2463] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cb958ffb8ba54460a13472fe93aba131 in term 0.
10:05:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.232352 11127 raft_consensus.cc:2463] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cb958ffb8ba54460a13472fe93aba131 in term 0.
10:05:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.232951 11196 leader_election.cc:304] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 34c6f1b7300a46919cb1515f6f721cbc, 4a5806654c4148fbb6e787f4ad65533a, cb958ffb8ba54460a13472fe93aba131; no voters:
10:05:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.233328 11348 raft_consensus.cc:2798] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:05:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.233460 11348 raft_consensus.cc:491] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:05:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.233562 11348 raft_consensus.cc:3054] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Advancing to term 1
10:05:32.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.235217 11348 raft_consensus.cc:513] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.235646 11348 leader_election.cc:290] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [CANDIDATE]: Term 1 election: Requested vote from peers 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803), 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.235954 10987 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "59ba63bafba443cdb09fba9d4814185f" candidate_uuid: "cb958ffb8ba54460a13472fe93aba131" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a5806654c4148fbb6e787f4ad65533a"
10:05:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.236091 10987 raft_consensus.cc:3054] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Advancing to term 1
10:05:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.236115 11127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "59ba63bafba443cdb09fba9d4814185f" candidate_uuid: "cb958ffb8ba54460a13472fe93aba131" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34c6f1b7300a46919cb1515f6f721cbc"
10:05:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.236295 11127 raft_consensus.cc:3054] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Advancing to term 1
10:05:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.238210 10987 raft_consensus.cc:2463] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cb958ffb8ba54460a13472fe93aba131 in term 1.
10:05:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.238410 11127 raft_consensus.cc:2463] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cb958ffb8ba54460a13472fe93aba131 in term 1.
10:05:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.238528 11194 leader_election.cc:304] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a5806654c4148fbb6e787f4ad65533a, cb958ffb8ba54460a13472fe93aba131; no voters:
10:05:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.238806 11348 raft_consensus.cc:2798] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 1 FOLLOWER]: Leader election won for term 1
10:05:32.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.239177 11348 raft_consensus.cc:695] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [term 1 LEADER]: Becoming Leader. State: Replica: cb958ffb8ba54460a13472fe93aba131, State: Running, Role: LEADER
10:05:32.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.239521 11348 consensus_queue.cc:237] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.244382 10624 catalog_manager.cc:5526] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 reported cstate change: term changed from 0 to 1, leader changed from <none> to cb958ffb8ba54460a13472fe93aba131 (127.10.85.131). New cstate: current_term: 1 leader_uuid: "cb958ffb8ba54460a13472fe93aba131" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } health_report { overall_health: HEALTHY } } }
10:05:32.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.255002 11346 raft_consensus.cc:491] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:05:32.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.255162 11317 tablet.cc:2367] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:05:32.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.255164 11346 raft_consensus.cc:513] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.256733 11346 leader_election.cc:290] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803), cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.264359 11269 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6ae46b0d157a4cd98d06940e64ffe5af" candidate_uuid: "34c6f1b7300a46919cb1515f6f721cbc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb958ffb8ba54460a13472fe93aba131" is_pre_election: true
10:05:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.264664 11269 raft_consensus.cc:2463] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34c6f1b7300a46919cb1515f6f721cbc in term 0.
10:05:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.265235 11055 leader_election.cc:304] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34c6f1b7300a46919cb1515f6f721cbc, cb958ffb8ba54460a13472fe93aba131; no voters:
10:05:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.265601 11346 raft_consensus.cc:2798] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Leader pre-election won for term 1
10:05:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.265722 11346 raft_consensus.cc:491] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:05:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.265843 11346 raft_consensus.cc:3054] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Advancing to term 1
10:05:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.268411 10987 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6ae46b0d157a4cd98d06940e64ffe5af" candidate_uuid: "34c6f1b7300a46919cb1515f6f721cbc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a5806654c4148fbb6e787f4ad65533a" is_pre_election: true
10:05:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.268564 11346 raft_consensus.cc:513] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.268656 10987 raft_consensus.cc:2463] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34c6f1b7300a46919cb1515f6f721cbc in term 0.
10:05:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.269193 11346 leader_election.cc:290] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [CANDIDATE]: Term 1 election: Requested vote from peers 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803), cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.269454 10987 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6ae46b0d157a4cd98d06940e64ffe5af" candidate_uuid: "34c6f1b7300a46919cb1515f6f721cbc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a5806654c4148fbb6e787f4ad65533a"
10:05:32.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.269658 10987 raft_consensus.cc:3054] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Advancing to term 1
10:05:32.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.269853 11269 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6ae46b0d157a4cd98d06940e64ffe5af" candidate_uuid: "34c6f1b7300a46919cb1515f6f721cbc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb958ffb8ba54460a13472fe93aba131"
10:05:32.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.270013 11269 raft_consensus.cc:3054] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Advancing to term 1
10:05:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.272218 10987 raft_consensus.cc:2463] T 6ae46b0d157a4cd98d06940e64ffe5af P 4a5806654c4148fbb6e787f4ad65533a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34c6f1b7300a46919cb1515f6f721cbc in term 1.
10:05:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.272564 11053 leader_election.cc:304] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34c6f1b7300a46919cb1515f6f721cbc, 4a5806654c4148fbb6e787f4ad65533a; no voters:
10:05:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.272634 11269 raft_consensus.cc:2463] T 6ae46b0d157a4cd98d06940e64ffe5af P cb958ffb8ba54460a13472fe93aba131 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34c6f1b7300a46919cb1515f6f721cbc in term 1.
10:05:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.272830 11346 raft_consensus.cc:2798] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 1 FOLLOWER]: Leader election won for term 1
10:05:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.273272 11346 raft_consensus.cc:695] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [term 1 LEADER]: Becoming Leader. State: Replica: 34c6f1b7300a46919cb1515f6f721cbc, State: Running, Role: LEADER
10:05:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.273715 11346 consensus_queue.cc:237] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } }
10:05:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.278748 10624 catalog_manager.cc:5526] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc reported cstate change: term changed from 0 to 1, leader changed from <none> to 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130). New cstate: current_term: 1 leader_uuid: "34c6f1b7300a46919cb1515f6f721cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } health_report { overall_health: HEALTHY } } }
10:05:32.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.321048 11175 tablet.cc:2367] T 59ba63bafba443cdb09fba9d4814185f P 34c6f1b7300a46919cb1515f6f721cbc: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.334823 10624 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55646:
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:05:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.336030 10624 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:05:32.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.349712 10966 tablet_service.cc:1467] Processing CreateTablet for tablet 03f537e9861247ecaa40c36fce6bcd91 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:05:32.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.350566 10966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 03f537e9861247ecaa40c36fce6bcd91. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.352109 11104 tablet_service.cc:1467] Processing CreateTablet for tablet 03f537e9861247ecaa40c36fce6bcd91 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:05:32.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.352926 11104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 03f537e9861247ecaa40c36fce6bcd91. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.355121 11340 tablet_bootstrap.cc:492] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: Bootstrap starting.
10:05:32.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.356534 11340 tablet_bootstrap.cc:654] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: Neither blocks nor log segments found. Creating new log.
10:05:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.358201 11340 tablet_bootstrap.cc:492] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: No bootstrap required, opened a new log
10:05:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.358350 11340 ts_tablet_manager.cc:1397] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
10:05:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.358877 11340 raft_consensus.cc:357] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.359035 11340 raft_consensus.cc:383] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.359279 11341 tablet_bootstrap.cc:492] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: Bootstrap starting.
10:05:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.360119 11246 tablet_service.cc:1467] Processing CreateTablet for tablet 03f537e9861247ecaa40c36fce6bcd91 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:05:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.360831 11341 tablet_bootstrap.cc:654] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: Neither blocks nor log segments found. Creating new log.
10:05:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.360899 11246 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 03f537e9861247ecaa40c36fce6bcd91. 1 dirs total, 0 dirs full, 0 dirs failed
10:05:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.359102 11340 raft_consensus.cc:738] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a5806654c4148fbb6e787f4ad65533a, State: Initialized, Role: FOLLOWER
10:05:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.361588 11035 tablet.cc:2367] T 59ba63bafba443cdb09fba9d4814185f P 4a5806654c4148fbb6e787f4ad65533a: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:05:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.364796 11340 consensus_queue.cc:260] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.365190 11341 tablet_bootstrap.cc:492] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: No bootstrap required, opened a new log
10:05:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.365367 11341 ts_tablet_manager.cc:1397] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: Time spent bootstrapping tablet: real 0.006s user 0.005s sys 0.000s
10:05:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.365631 11347 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.365970 11347 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.366499 11341 raft_consensus.cc:357] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.366729 11340 ts_tablet_manager.cc:1428] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: Time spent starting tablet: real 0.008s user 0.000s sys 0.002s
10:05:32.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.367424 11347 raft_consensus.cc:491] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:05:32.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.367594 11347 raft_consensus.cc:513] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.368057 11341 raft_consensus.cc:383] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.368212 11341 raft_consensus.cc:738] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34c6f1b7300a46919cb1515f6f721cbc, State: Initialized, Role: FOLLOWER
10:05:32.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.368588 11341 consensus_queue.cc:260] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.369072 11346 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.369314 11346 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.369436 11347 leader_election.cc:290] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325), cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.372668 11341 ts_tablet_manager.cc:1428] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: Time spent starting tablet: real 0.007s user 0.001s sys 0.000s
10:05:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.376032 11342 tablet_bootstrap.cc:492] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: Bootstrap starting.
10:05:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.377707 11342 tablet_bootstrap.cc:654] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: Neither blocks nor log segments found. Creating new log.
10:05:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.379824 11342 tablet_bootstrap.cc:492] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: No bootstrap required, opened a new log
10:05:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.379979 11342 ts_tablet_manager.cc:1397] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.001s
10:05:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.380645 11342 raft_consensus.cc:357] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.381271 11342 raft_consensus.cc:383] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:05:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.381340 11342 raft_consensus.cc:738] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb958ffb8ba54460a13472fe93aba131, State: Initialized, Role: FOLLOWER
10:05:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.381656 11342 consensus_queue.cc:260] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.382632 11348 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.382786 11342 ts_tablet_manager.cc:1428] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
10:05:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.382922 11348 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.389987 11127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03f537e9861247ecaa40c36fce6bcd91" candidate_uuid: "4a5806654c4148fbb6e787f4ad65533a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34c6f1b7300a46919cb1515f6f721cbc" is_pre_election: true
10:05:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.390208 11127 raft_consensus.cc:2463] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4a5806654c4148fbb6e787f4ad65533a in term 0.
10:05:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.390422 11269 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03f537e9861247ecaa40c36fce6bcd91" candidate_uuid: "4a5806654c4148fbb6e787f4ad65533a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb958ffb8ba54460a13472fe93aba131" is_pre_election: true
10:05:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.390592 11269 raft_consensus.cc:2463] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4a5806654c4148fbb6e787f4ad65533a in term 0.
10:05:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.390609 10911 leader_election.cc:304] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34c6f1b7300a46919cb1515f6f721cbc, 4a5806654c4148fbb6e787f4ad65533a; no voters:
10:05:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.390997 11347 raft_consensus.cc:2798] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Leader pre-election won for term 1
10:05:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.391124 11347 raft_consensus.cc:491] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:05:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.391214 11347 raft_consensus.cc:3054] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 0 FOLLOWER]: Advancing to term 1
10:05:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.393034 11347 raft_consensus.cc:513] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.393525 11347 leader_election.cc:290] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [CANDIDATE]: Term 1 election: Requested vote from peers 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325), cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.393908 11127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03f537e9861247ecaa40c36fce6bcd91" candidate_uuid: "4a5806654c4148fbb6e787f4ad65533a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34c6f1b7300a46919cb1515f6f721cbc"
10:05:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.393954 11269 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03f537e9861247ecaa40c36fce6bcd91" candidate_uuid: "4a5806654c4148fbb6e787f4ad65533a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb958ffb8ba54460a13472fe93aba131"
10:05:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.394055 11127 raft_consensus.cc:3054] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 0 FOLLOWER]: Advancing to term 1
10:05:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.394114 11269 raft_consensus.cc:3054] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 0 FOLLOWER]: Advancing to term 1
10:05:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.395609 11127 raft_consensus.cc:2463] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4a5806654c4148fbb6e787f4ad65533a in term 1.
10:05:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.395735 11269 raft_consensus.cc:2463] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4a5806654c4148fbb6e787f4ad65533a in term 1.
10:05:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.395921 10911 leader_election.cc:304] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34c6f1b7300a46919cb1515f6f721cbc, 4a5806654c4148fbb6e787f4ad65533a; no voters:
10:05:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.396165 11347 raft_consensus.cc:2798] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 1 FOLLOWER]: Leader election won for term 1
10:05:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.396560 11347 raft_consensus.cc:695] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [term 1 LEADER]: Becoming Leader. State: Replica: 4a5806654c4148fbb6e787f4ad65533a, State: Running, Role: LEADER
10:05:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.397465 11347 consensus_queue.cc:237] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } }
10:05:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.400566 11372 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: TxnStatusTablet state changed. Reason: New leader 4a5806654c4148fbb6e787f4ad65533a. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.400890 11372 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: This TxnStatusTablet replica's current role is: LEADER
10:05:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.401531 11374 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
10:05:32.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.401921 11374 txn_status_manager.cc:930] Loading transaction status metadata into memory...
10:05:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.402197 10623 catalog_manager.cc:5526] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a reported cstate change: term changed from 0 to 1, leader changed from <none> to 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129). New cstate: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } health_report { overall_health: UNKNOWN } } }
10:05:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.436306 11127 raft_consensus.cc:1270] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc [term 1 FOLLOWER]: Refusing update from remote peer 4a5806654c4148fbb6e787f4ad65533a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:05:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.436352 11269 raft_consensus.cc:1270] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131 [term 1 FOLLOWER]: Refusing update from remote peer 4a5806654c4148fbb6e787f4ad65533a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:05:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.437135 11372 consensus_queue.cc:1035] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [LEADER]: Connected to new peer: Peer: permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.437464 11347 consensus_queue.cc:1035] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a [LEADER]: Connected to new peer: Peer: permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.454375 11348 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: TxnStatusTablet state changed. Reason: New leader 4a5806654c4148fbb6e787f4ad65533a. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.454733 11348 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.455978 11346 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: TxnStatusTablet state changed. Reason: New leader 4a5806654c4148fbb6e787f4ad65533a. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.456274 11346 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.474941 11346 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.475188 11346 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 34c6f1b7300a46919cb1515f6f721cbc: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.478341 11348 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.478569 11348 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P cb958ffb8ba54460a13472fe93aba131: This TxnStatusTablet replica's current role is: FOLLOWER
10:05:32.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.481093 11347 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.481382 11347 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: This TxnStatusTablet replica's current role is: LEADER
10:05:32.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.482589 11347 tablet_replica.cc:440] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4a5806654c4148fbb6e787f4ad65533a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 } } peers { permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 } } peers { permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 } } }
10:05:32.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.483598 11347 tablet_replica.cc:443] T 03f537e9861247ecaa40c36fce6bcd91 P 4a5806654c4148fbb6e787f4ad65533a: This TxnStatusTablet replica's current role is: LEADER
10:05:32.487 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
10:05:32.489 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dd388c02673449b9d85313eaa39f243
10:05:32.489 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.85.190:43483
10:05:32.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:05:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.490952 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 10593
10:05:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.492074 10664 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.616497 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 10593
10:05:32.647 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.85.190:43483(127.10.85.190:43483)] handling channelInactive
10:05:32.647 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.190:43483(127.10.85.190:43483)] cleaning up while in state READY due to: connection closed
10:05:32.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:05:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.703478 11348 consensus_queue.cc:1035] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [LEADER]: Connected to new peer: Peer: permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" member_type: VOTER last_known_addr { host: "127.10.85.130" port: 45325 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.706362 11173 heartbeater.cc:646] Failed to heartbeat to 127.10.85.190:43483 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.709659 11388 raft_consensus.cc:491] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d989751ff71c4a4b81b2f08770595162)
10:05:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.710510 11388 raft_consensus.cc:513] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.710677 11315 heartbeater.cc:646] Failed to heartbeat to 127.10.85.190:43483 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.710724 11348 consensus_queue.cc:1035] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.715797 11388 leader_election.cc:290] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:32.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.716539 10796 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "585db50113304f22ac16c7e112fcddab" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" is_pre_election: true
10:05:32.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.716760 11033 heartbeater.cc:646] Failed to heartbeat to 127.10.85.190:43483 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.719156 10691 leader_election.cc:336] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.719408 10691 leader_election.cc:304] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 585db50113304f22ac16c7e112fcddab; no voters: d989751ff71c4a4b81b2f08770595162, f4e788cbdfb84919a6ecb74d02dbd0b0
10:05:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.719938 11388 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
10:05:32.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.788539 11392 raft_consensus.cc:491] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d989751ff71c4a4b81b2f08770595162)
10:05:32.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.788731 11392 raft_consensus.cc:513] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:32.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.790052 11392 leader_election.cc:290] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575)
10:05:32.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.790685 10713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "585db50113304f22ac16c7e112fcddab" is_pre_election: true
10:05:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.790958 10713 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f4e788cbdfb84919a6ecb74d02dbd0b0 in term 1.
10:05:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.791172 10772 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.791437 10770 leader_election.cc:304] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 585db50113304f22ac16c7e112fcddab, f4e788cbdfb84919a6ecb74d02dbd0b0; no voters:
10:05:32.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.792724 11392 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Leader pre-election won for term 2
10:05:32.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.792884 11392 raft_consensus.cc:491] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Starting leader election (detected failure of leader d989751ff71c4a4b81b2f08770595162)
10:05:32.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.792992 11392 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 1 FOLLOWER]: Advancing to term 2
10:05:32.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.794499 11346 consensus_queue.cc:1035] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [LEADER]: Connected to new peer: Peer: permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" member_type: VOTER last_known_addr { host: "127.10.85.129" port: 43803 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:32.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.798328 10772 leader_election.cc:336] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.799691 11392 raft_consensus.cc:513] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:32.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.800271 11392 leader_election.cc:290] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 2 election: Requested vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575)
10:05:32.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.800855 10713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "585db50113304f22ac16c7e112fcddab"
10:05:32.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.801055 10713 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 1 FOLLOWER]: Advancing to term 2
10:05:32.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.802286 10772 leader_election.cc:336] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111)
10:05:32.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.803727 10713 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f4e788cbdfb84919a6ecb74d02dbd0b0 in term 2.
10:05:32.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.804396 10770 leader_election.cc:304] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 585db50113304f22ac16c7e112fcddab, f4e788cbdfb84919a6ecb74d02dbd0b0; no voters: d989751ff71c4a4b81b2f08770595162
10:05:32.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.804888 11392 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 2 FOLLOWER]: Leader election won for term 2
10:05:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.817571 11392 raft_consensus.cc:695] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 2 LEADER]: Becoming Leader. State: Replica: f4e788cbdfb84919a6ecb74d02dbd0b0, State: Running, Role: LEADER
10:05:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.817970 11399 consensus_queue.cc:1035] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc [LEADER]: Connected to new peer: Peer: permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" member_type: VOTER last_known_addr { host: "127.10.85.131" port: 32801 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:05:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.818369 11392 consensus_queue.cc:237] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:32.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.820914 11398 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4e788cbdfb84919a6ecb74d02dbd0b0. Latest consensus state: current_term: 2 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:32.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.821254 11398 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: LEADER
10:05:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.823767 11401 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:05:32.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.826485 11401 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d]
10:05:32.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.827052 11401 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]
10:05:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.828528 11401 tablet_loader.cc:96] loaded metadata for tablet 03f537e9861247ecaa40c36fce6bcd91 (table kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d])
10:05:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.828928 11401 tablet_loader.cc:96] loaded metadata for tablet 59ba63bafba443cdb09fba9d4814185f (table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243])
10:05:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.829195 11401 tablet_loader.cc:96] loaded metadata for tablet 6ae46b0d157a4cd98d06940e64ffe5af (table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243])
10:05:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.829370 11401 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:05:32.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.830005 11401 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:32.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.830083 11401 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:05:32.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.831013 11401 catalog_manager.cc:1505] Loading token signing keys...
10:05:32.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.831744 11401 catalog_manager.cc:5910] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Loaded TSK: 0
10:05:32.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.832441 11401 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:05:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:32.897641 10772 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Couldn't send request to peer d989751ff71c4a4b81b2f08770595162. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:05:32.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.933013 10713 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Refusing update from remote peer f4e788cbdfb84919a6ecb74d02dbd0b0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
10:05:32.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.933626 11398 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
10:05:32.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.934818 11388 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4e788cbdfb84919a6ecb74d02dbd0b0. Latest consensus state: current_term: 2 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:32.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.934988 11388 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: FOLLOWER
10:05:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.936290 11392 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.936518 11392 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: LEADER
10:05:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.936586 11388 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:32.936769 11388 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: FOLLOWER
10:05:32.992 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.190:43483(127.10.85.190:43483)] connecting to peer
10:05:32.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483
10:05:32.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.190:43483(127.10.85.190:43483)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483
10:05:32.993 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.10.85.190:43483(127.10.85.190:43483) for tablet Kudu Master: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483
10:05:33.012 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.190:43483(127.10.85.190:43483)] connecting to peer
10:05:33.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483
10:05:33.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.190:43483(127.10.85.190:43483)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483
10:05:33.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.190:43483: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483
10:05:33.015 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:05:33.015 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] sending RPC to server master-127.10.85.190:43483, [1ms] received response from server master-127.10.85.190:43483: Network error: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: Network error: Failed to connect to peer master-127.10.85.190:43483(127.10.85.190:43483): Connection refused: /127.10.85.190:43483, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, deferred=Deferred@1846129507(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@61ee571, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@2477b2f)) after lookup
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:33.045228 10785 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40164:
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:05:33.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:33.047129 10785 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:33.372222 10772 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Couldn't send request to peer d989751ff71c4a4b81b2f08770595162. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:05:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:33.924983 10772 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Couldn't send request to peer d989751ff71c4a4b81b2f08770595162. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:05:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:34.332885 11314 heartbeater.cc:501] Master 127.10.85.188:35823 was elected leader, sending a full tablet report...
10:05:34.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:34.364240 11172 heartbeater.cc:501] Master 127.10.85.188:35823 was elected leader, sending a full tablet report...
10:05:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.419711 10772 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Couldn't send request to peer d989751ff71c4a4b81b2f08770595162. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:05:34.651 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6ae46b0d157a4cd98d06940e64ffe5af for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
10:05:34.652 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 59ba63bafba443cdb09fba9d4814185f for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
10:05:34.652 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=6ae46b0d157a4cd98d06940e64ffe5af@[34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325)[L],4a5806654c4148fbb6e787f4ad65533a(127.10.85.129:43803),cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=59ba63bafba443cdb09fba9d4814185f@[34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325),4a5806654c4148fbb6e787f4ad65533a(127.10.85.129:43803),cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
10:05:34.652 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=1), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.85.188:35823, [1ms] Sub RPC GetTableLocations: received response from server master-127.10.85.188:35823: OK, deferred=null) after lookup
10:05:34.652 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325)] connecting to peer
10:05:34.653 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325)] Successfully connected to peer
10:05:34.663 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1520a0d1, L:/127.0.0.1:34076 - R:/127.10.85.130:45325] using TOKEN/null
10:05:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.667394 11100 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.10.85.190:43483)
10:05:34.697 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.85.190:43483
10:05:34.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:34.698534 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/logs
10:05:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/info.pb
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.190:43483
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.190
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39191
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:05:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:05:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:34.822876 11032 heartbeater.cc:501] Master 127.10.85.188:35823 was elected leader, sending a full tablet report...
10:05:34.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.918901 10772 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483): Couldn't send request to peer d989751ff71c4a4b81b2f08770595162. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.190:43483: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:05:34.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.992839 11445 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:34.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.993075 11445 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:34.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.993155 11445 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:34.999994 11445 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.000087 11445 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.000121 11445 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.000185 11445 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.000222 11445 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.000270 11445 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.000304 11445 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45569/jwks.json
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.009045 11445 master_runner.cc:386] Master server non-default flags:
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.190:43483
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/info.pb
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.190
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39191
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/logs
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:05:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.010632 11445 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:35.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.012751 11445 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.023360 11452 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.023891 11451 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.024408 11454 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.025300 11445 server_base.cc:1034] running on GCE node
10:05:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.025897 11445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:35.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.027081 11445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:35.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.028286 11445 hybrid_clock.cc:648] HybridClock initialized: now 1736935535028233 us; error 30 us; skew 500 ppm
10:05:35.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.028607 11445 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:35.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.038259 11445 webserver.cc:458] Webserver started at http://127.10.85.190:39191/ using document root <none> and password file <none>
10:05:35.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.038884 11445 fs_manager.cc:362] Metadata directory not provided
10:05:35.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.038987 11445 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.043449 11445 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
10:05:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.045751 11460 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.046770 11445 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
10:05:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.046905 11445 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d989751ff71c4a4b81b2f08770595162"
10:05:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:29 on dist-test-slave-jkng"
10:05:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.047694 11445 fs_report.cc:389] FS layout report
10:05:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/wal
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/data
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.065888 11445 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.066581 11445 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.066840 11445 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.085623 11445 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.190:43483
10:05:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.085670 11522 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.190:43483 every 8 connection(s)
10:05:35.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.086845 11445 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-0/data/info.pb
10:05:35.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.092332 11523 sys_catalog.cc:263] Verifying existing consensus state
10:05:35.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.093549 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 11445
10:05:35.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:05:35.093 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.85.188:35823
10:05:35.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:05:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.094238 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 10754
10:05:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.095196 10826 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.095299 11523 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Bootstrap starting.
10:05:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.113168 11523 log.cc:826] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Log is configured to *not* fsync() on all Append() calls
10:05:35.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.127388 11523 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:05:35.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.127925 11523 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: Bootstrap complete.
10:05:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.135912 11523 raft_consensus.cc:357] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.136605 11523 raft_consensus.cc:738] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d989751ff71c4a4b81b2f08770595162, State: Initialized, Role: FOLLOWER
10:05:35.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.137142 11523 consensus_queue.cc:260] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:35.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.137948 11527 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:35.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.138267 11527 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:35.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.138949 11523 sys_catalog.cc:564] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: configured and running, proceeding with master startup.
10:05:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.148088 11538 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.148192 11538 catalog_manager.cc:1553] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: loading cluster ID for follower catalog manager: success
10:05:35.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.152925 11538 catalog_manager.cc:1575] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: acquiring CA information for follower catalog manager: success
10:05:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.155768 11538 catalog_manager.cc:1603] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:05:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.219373 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 10754
10:05:35.240 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:05:35.240 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.85.188:35823(127.10.85.188:35823)] handling channelInactive
10:05:35.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.188:35823(127.10.85.188:35823)] cleaning up while in state READY due to: connection closed
10:05:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.240746 11540 negotiation.cc:337] Failed RPC negotiation. Trace:
10:05:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.181049 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.10.85.188:35823 (local address 127.0.0.1:46622)
10:05:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.181178 (+ 129us) negotiation.cc:107] Waiting for socket to connect
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.181193 (+ 15us) client_negotiation.cc:174] Beginning negotiation
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.181276 (+ 83us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.218915 (+ 37639us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.218921 (+ 6us) client_negotiation.cc:394] Received NEGOTIATE response from server
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.218968 (+ 47us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.219408 (+ 440us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.219423 (+ 15us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.238927 (+ 19504us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":43}
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.240774 11541 negotiation.cc:337] Failed RPC negotiation. Trace:
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.218680 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53802 (local address 127.10.85.190:43483)
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220328 (+ 1648us) server_negotiation.cc:206] Beginning negotiation
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220333 (+ 5us) server_negotiation.cc:399] Waiting for connection header
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220350 (+ 17us) server_negotiation.cc:407] Connection header received
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220401 (+ 51us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220405 (+ 4us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220466 (+ 61us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.220535 (+ 69us) server_negotiation.cc:226] Negotiated authn=CERTIFICATE
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:35.238898 (+ 18363us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53802: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:05:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1539}
10:05:35.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.241175 11445 master_runner.cc:186] Error getting master registration for 127.10.85.188:35823: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
10:05:35.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.245512 11445 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:05:35.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.339241 11315 heartbeater.cc:346] Connected to a master server at 127.10.85.190:43483
10:05:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.342275 11478 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" instance_seqno: 1736935531991487) as {username='slave'} at 127.10.85.131:35115; Asking this server to re-register.
10:05:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.342948 11315 heartbeater.cc:463] Registering TS with master...
10:05:35.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.344276 11478 ts_manager.cc:194] Registered new tserver with Master: cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.371332 11173 heartbeater.cc:346] Connected to a master server at 127.10.85.190:43483
10:05:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.372545 11478 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" instance_seqno: 1736935531556451) as {username='slave'} at 127.10.85.130:46427; Asking this server to re-register.
10:05:35.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.373013 11173 heartbeater.cc:463] Registering TS with master...
10:05:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.373957 11478 ts_manager.cc:194] Registered new tserver with Master: 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.382532 11527 raft_consensus.cc:491] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:05:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.382781 11527 raft_consensus.cc:513] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.384809 11527 leader_election.cc:290] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.385118 10713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d989751ff71c4a4b81b2f08770595162" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "585db50113304f22ac16c7e112fcddab" is_pre_election: true
10:05:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.385455 11463 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.386816 11463 leader_election.cc:336] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.386945 11463 leader_election.cc:304] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d989751ff71c4a4b81b2f08770595162; no voters: 585db50113304f22ac16c7e112fcddab, f4e788cbdfb84919a6ecb74d02dbd0b0
10:05:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.387236 11527 raft_consensus.cc:2743] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
10:05:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.518823 11550 raft_consensus.cc:491] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Starting pre-election (detected failure of leader f4e788cbdfb84919a6ecb74d02dbd0b0)
10:05:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.519004 11550 raft_consensus.cc:513] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.519718 11550 leader_election.cc:290] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.521011 10691 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111) [suppressed 5 similar messages]
10:05:35.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.522222 10691 leader_election.cc:336] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.525465 11488 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "585db50113304f22ac16c7e112fcddab" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "d989751ff71c4a4b81b2f08770595162" is_pre_election: true
10:05:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.525878 11488 raft_consensus.cc:2463] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 585db50113304f22ac16c7e112fcddab in term 1.
10:05:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.526257 10691 leader_election.cc:304] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 585db50113304f22ac16c7e112fcddab, d989751ff71c4a4b81b2f08770595162; no voters: f4e788cbdfb84919a6ecb74d02dbd0b0
10:05:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.526502 11550 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Leader pre-election won for term 3
10:05:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.526607 11550 raft_consensus.cc:491] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Starting leader election (detected failure of leader f4e788cbdfb84919a6ecb74d02dbd0b0)
10:05:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.526666 11550 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 2 FOLLOWER]: Advancing to term 3
10:05:35.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.528669 11550 raft_consensus.cc:513] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.529138 11550 leader_election.cc:290] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 3 election: Requested vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.529541 11488 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "585db50113304f22ac16c7e112fcddab" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "d989751ff71c4a4b81b2f08770595162"
10:05:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.529729 11488 raft_consensus.cc:3054] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 1 FOLLOWER]: Advancing to term 3
10:05:35.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.530333 10691 leader_election.cc:336] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111)
10:05:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.533788 11488 raft_consensus.cc:2463] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 585db50113304f22ac16c7e112fcddab in term 3.
10:05:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.534096 10691 leader_election.cc:304] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 585db50113304f22ac16c7e112fcddab, d989751ff71c4a4b81b2f08770595162; no voters: f4e788cbdfb84919a6ecb74d02dbd0b0
10:05:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.534327 11550 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 3 FOLLOWER]: Leader election won for term 3
10:05:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.534642 11550 raft_consensus.cc:695] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [term 3 LEADER]: Becoming Leader. State: Replica: 585db50113304f22ac16c7e112fcddab, State: Running, Role: LEADER
10:05:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.534965 11550 consensus_queue.cc:237] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.536326 11554 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: New leader 585db50113304f22ac16c7e112fcddab. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.536547 11554 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: LEADER
10:05:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.537096 11556 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:05:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.539374 11556 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d]
10:05:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.539928 11556 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]
10:05:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.541580 11556 tablet_loader.cc:96] loaded metadata for tablet 03f537e9861247ecaa40c36fce6bcd91 (table kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d])
10:05:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.542080 11556 tablet_loader.cc:96] loaded metadata for tablet 59ba63bafba443cdb09fba9d4814185f (table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243])
10:05:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.542474 11556 tablet_loader.cc:96] loaded metadata for tablet 6ae46b0d157a4cd98d06940e64ffe5af (table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243])
10:05:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.542745 11556 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:05:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.543602 11556 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:35.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.543844 11556 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:05:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.545020 11556 catalog_manager.cc:1505] Loading token signing keys...
10:05:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.545886 11556 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab: Loaded TSK: 0
10:05:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.546675 11556 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:05:35.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.631469 11488 raft_consensus.cc:1270] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 3 FOLLOWER]: Refusing update from remote peer 585db50113304f22ac16c7e112fcddab: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
10:05:35.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.632277 11554 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [LEADER]: Connected to new peer: Peer: permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
10:05:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.636086 11559 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.636417 11559 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.636417 11527 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 585db50113304f22ac16c7e112fcddab. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.636734 11527 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.637487 11527 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.637686 11527 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.638525 11554 sys_catalog.cc:455] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.638754 11554 sys_catalog.cc:458] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab [sys.catalog]: This master's current role is: LEADER
10:05:35.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.639420 10691 consensus_peers.cc:487] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab -> Peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Couldn't send request to peer f4e788cbdfb84919a6ecb74d02dbd0b0. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:05:35.732 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.188:35823(127.10.85.188:35823)] connecting to peer
10:05:35.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823
10:05:35.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.188:35823(127.10.85.188:35823)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823
10:05:35.733 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.10.85.188:35823(127.10.85.188:35823) for tablet Kudu Master: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823
10:05:35.752 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.190:43483(127.10.85.190:43483)] connecting to peer
10:05:35.752 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.188:35823(127.10.85.188:35823)] connecting to peer
10:05:35.753 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.85.190:43483(127.10.85.190:43483)] Successfully connected to peer
10:05:35.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823
10:05:35.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.188:35823(127.10.85.188:35823)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823
10:05:35.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.188:35823: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823
10:05:35.755 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:05:35.755 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [1ms] received response from server master-127.10.85.188:35823: Network error: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: Network error: Failed to connect to peer master-127.10.85.188:35823(127.10.85.188:35823): Connection refused: /127.10.85.188:35823, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: OK, deferred=Deferred@1257025559(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@3db02890, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@2e4cb99a)) after lookup
10:05:35.765 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd34ed4b5, L:/127.0.0.1:53832 - R:/127.10.85.190:43483] using TOKEN/null
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.797613 10704 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46646:
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:05:35.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:35.802974 10704 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:05:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.814162 11033 heartbeater.cc:346] Connected to a master server at 127.10.85.190:43483
10:05:35.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.815310 11478 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" instance_seqno: 1736935531098583) as {username='slave'} at 127.10.85.129:49375; Asking this server to re-register.
10:05:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.815908 11033 heartbeater.cc:463] Registering TS with master...
10:05:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:35.817222 11478 ts_manager.cc:194] Registered new tserver with Master: 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803)
10:05:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:36.104280 10691 consensus_peers.cc:487] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab -> Peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Couldn't send request to peer f4e788cbdfb84919a6ecb74d02dbd0b0. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:05:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:36.612016 10691 consensus_peers.cc:487] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab -> Peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Couldn't send request to peer f4e788cbdfb84919a6ecb74d02dbd0b0. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:05:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:36.813150 11034 heartbeater.cc:501] Master 127.10.85.189:33575 was elected leader, sending a full tablet report...
10:05:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.080957 10691 consensus_peers.cc:487] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab -> Peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Couldn't send request to peer f4e788cbdfb84919a6ecb74d02dbd0b0. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:05:37.243 [DEBUG - Time-limited test] (Connection.java:748) [peer cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)] connecting to peer
10:05:37.244 [DEBUG - kudu-nio-1] (Connection.java:757) [peer cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)] Successfully connected to peer
10:05:37.254 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x303f4512, L:/127.0.0.1:56342 - R:/127.10.85.131:32801] using TOKEN/null
10:05:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.262050 11241 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.10.85.190:43483)
10:05:37.289 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.85.188:35823
10:05:37.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
10:05:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.290675 10582 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu
10:05:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data
10:05:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/logs
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/info.pb
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.188:35823
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.188
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45959
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:05:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:05:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.336273 11316 heartbeater.cc:501] Master 127.10.85.189:33575 was elected leader, sending a full tablet report...
10:05:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.368774 11174 heartbeater.cc:501] Master 127.10.85.189:33575 was elected leader, sending a full tablet report...
10:05:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.559689 10691 consensus_peers.cc:487] T 00000000000000000000000000000000 P 585db50113304f22ac16c7e112fcddab -> Peer f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823): Couldn't send request to peer f4e788cbdfb84919a6ecb74d02dbd0b0. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.188:35823: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:05:37.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.589715 11598 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:05:37.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.589942 11598 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:05:37.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.590019 11598 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596410 11598 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596484 11598 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596516 11598 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596558 11598 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596590 11598 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596634 11598 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:05:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.596669 11598 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45569/jwks.json
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.605046 11598 master_runner.cc:386] Master server non-default flags:
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.85.148:37699
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.85.190:43483,127.10.85.189:33575,127.10.85.188:35823
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.85.188:35823
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45569/jwks.json
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/info.pb
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.85.188
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45959
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/logs
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 09:25:59 UTC on 24a791456cd2
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4187
10:05:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:05:37.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.606186 11598 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:05:37.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.607574 11598 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:05:37.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.617180 11605 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.618043 11608 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.618688 11606 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:05:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.619356 11598 server_base.cc:1034] running on GCE node
10:05:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.620054 11598 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:05:37.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.621304 11598 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:05:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.622491 11598 hybrid_clock.cc:648] HybridClock initialized: now 1736935537622445 us; error 37 us; skew 500 ppm
10:05:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.622812 11598 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:05:37.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.624806 11598 webserver.cc:458] Webserver started at http://127.10.85.188:45959/ using document root <none> and password file <none>
10:05:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.625474 11598 fs_manager.cc:362] Metadata directory not provided
10:05:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.625599 11598 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:05:37.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.630302 11598 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.000s sys 0.004s
10:05:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.632627 11614 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:05:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.633802 11598 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.633934 11598 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data,/tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0"
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:05:30 on dist-test-slave-jkng"
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.634579 11598 fs_report.cc:389] FS layout report
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/wal
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/data
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:05:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:05:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:05:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:05:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.660157 11598 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:05:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.660820 11598 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:05:37.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.661032 11598 kserver.cc:163] Server-wide thread pool size limit: 3276
10:05:37.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.679286 11598 rpc_server.cc:307] RPC server started. Bound to: 127.10.85.188:35823
10:05:37.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.679335 11675 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.85.188:35823 every 8 connection(s)
10:05:37.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.680462 11598 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkcvsP0/test-tmp/mini-kudu-cluster1004319422863447167/master-2/data/info.pb
10:05:37.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.684721 10582 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu as pid 11598
10:05:37.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:05:37.685 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.85.189:33575
10:05:37.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:05:37.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.685429 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 10671
10:05:37.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.686127 11676 sys_catalog.cc:263] Verifying existing consensus state
10:05:37.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.686374 10745 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.690418 11676 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Bootstrap starting.
10:05:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.704979 11676 log.cc:826] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Log is configured to *not* fsync() on all Append() calls
10:05:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.716511 11676 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:05:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.716923 11676 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Bootstrap complete.
10:05:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.723937 11676 raft_consensus.cc:357] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.724714 11676 raft_consensus.cc:738] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: f4e788cbdfb84919a6ecb74d02dbd0b0, State: Initialized, Role: FOLLOWER
10:05:37.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.725283 11676 consensus_queue.cc:260] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.725996 11680 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.726277 11680 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: FOLLOWER
10:05:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.727453 11676 sys_catalog.cc:564] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: configured and running, proceeding with master startup.
10:05:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.750669 11691 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.750857 11691 catalog_manager.cc:1553] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: loading cluster ID for follower catalog manager: success
10:05:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.754531 11691 catalog_manager.cc:1575] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: acquiring CA information for follower catalog manager: success
10:05:37.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.756848 11691 catalog_manager.cc:1603] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:05:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.838099 11641 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 2 FOLLOWER]: Advancing to term 3
10:05:37.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.844575 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 10671
10:05:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.866546 11032 heartbeater.cc:346] Connected to a master server at 127.10.85.188:35823
10:05:37.874 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.85.189:33575(127.10.85.189:33575)] handling channelInactive
10:05:37.875 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state READY due to: connection closed
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.875010 11693 negotiation.cc:337] Failed RPC negotiation. Trace:
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:37.760143 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.10.85.189:33575 (local address 127.0.0.1:46652)
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:37.760288 (+ 145us) negotiation.cc:107] Waiting for socket to connect
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:37.760304 (+ 16us) client_negotiation.cc:174] Beginning negotiation
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:37.760426 (+ 122us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:37.873942 (+113516us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":50}
10:05:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.875748 10908 connection.cc:537] client connection to 127.10.85.189:33575 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:05:37.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:05:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:37.876195 11616 connection.cc:441] server connection from 127.0.0.1:47100 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:47100 (request call id 0) could send its response
10:05:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.877621 11702 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.878038 11702 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: FOLLOWER
10:05:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.878057 11680 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 585db50113304f22ac16c7e112fcddab. Latest consensus state: current_term: 3 leader_uuid: "585db50113304f22ac16c7e112fcddab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.878548 11680 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: FOLLOWER
10:05:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.875792 11598 master_runner.cc:186] Error getting master registration for 127.10.85.189:33575: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
10:05:37.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.881809 11598 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:05:37.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.882602 11631 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4a5806654c4148fbb6e787f4ad65533a" instance_seqno: 1736935531098583) as {username='slave'} at 127.10.85.129:33919; Asking this server to re-register.
10:05:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.883267 11032 heartbeater.cc:463] Registering TS with master...
10:05:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:37.884790 11631 ts_manager.cc:194] Registered new tserver with Master: 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803)
10:05:37.892 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:37.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.893 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.10.85.189:33575(127.10.85.189:33575) for tablet Kudu Master: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:37.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.188:35823(127.10.85.188:35823)] connecting to peer
10:05:37.913 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.913 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.85.188:35823(127.10.85.188:35823)] Successfully connected to peer
10:05:37.923 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x97243e12, L:/127.0.0.1:47104 - R:/127.10.85.188:35823] using TOKEN/null
10:05:37.925 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:37.932 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:37.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:37.952 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:37.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:37.972 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:37.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:37.992 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:37.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:37.994 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:38.076 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:38.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.080 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:38.082 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] sending RPC to server master-127.10.85.189:33575, [1ms] received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [33ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [40ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [184ms] refreshing cache from master, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [187ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [187ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [188ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, deferred=Deferred@283301425(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@5d7ee95f -> (continuation of Deferred@911769464 after retry RPC after error@765921719) -> (continuation of Deferred@737280364 after retry RPC after error@23863644) -> (continuation of Deferred@1949969846 after retry RPC after error@1640331319) -> (continuation of Deferred@1518857218 after retry RPC after error@973164751) -> (continuation of Deferred@536059300 after retry RPC after error@1438468781), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@30bf4be4 -> (continuation of Deferred@911769464 after retry RPC after error@765921719) -> (continuation of Deferred@737280364 after retry RPC after error@23863644) -> (continuation of Deferred@1949969846 after retry RPC after error@1640331319) -> (continuation of Deferred@1518857218 after retry RPC after error@973164751) -> (continuation of Deferred@536059300 after retry RPC after error@1438468781)))
10:05:38.082 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] sending RPC to server master-127.10.85.189:33575, [1ms] received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [33ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [40ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [184ms] refreshing cache from master, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [187ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [187ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [188ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, deferred=Deferred@283301425(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@5d7ee95f -> (continuation of Deferred@911769464 after retry RPC after error@765921719) -> (continuation of Deferred@737280364 after retry RPC after error@23863644) -> (continuation of Deferred@1949969846 after retry RPC after error@1640331319) -> (continuation of Deferred@1518857218 after retry RPC after error@973164751) -> (continuation of Deferred@536059300 after retry RPC after error@1438468781), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@30bf4be4 -> (continuation of Deferred@911769464 after retry RPC after error@765921719) -> (continuation of Deferred@737280364 after retry RPC after error@23863644) -> (continuation of Deferred@1949969846 after retry RPC after error@1640331319) -> (continuation of Deferred@1518857218 after retry RPC after error@973164751) -> (continuation of Deferred@536059300 after retry RPC after error@1438468781)))
10:05:38.092 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:38.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.093 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.095 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:38.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:38.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.113 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.115 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:38.132 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:38.133 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.133 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.133 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.138038 11703 raft_consensus.cc:491] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 585db50113304f22ac16c7e112fcddab)
10:05:38.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.138406 11703 raft_consensus.cc:513] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:38.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.140300 11703 leader_election.cc:290] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575), f4e788cbdfb84919a6ecb74d02dbd0b0 (127.10.85.188:35823)
10:05:38.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.147209 11680 raft_consensus.cc:491] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 585db50113304f22ac16c7e112fcddab)
10:05:38.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.147404 11680 raft_consensus.cc:513] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:38.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.149439 11680 leader_election.cc:290] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575)
10:05:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.150148 11488 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "d989751ff71c4a4b81b2f08770595162" is_pre_election: true
10:05:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.150251 11615 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.150341 11488 raft_consensus.cc:2463] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f4e788cbdfb84919a6ecb74d02dbd0b0 in term 3.
10:05:38.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.150949 11617 leader_election.cc:304] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d989751ff71c4a4b81b2f08770595162, f4e788cbdfb84919a6ecb74d02dbd0b0; no voters:
10:05:38.152 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:38.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.153 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.151388 11680 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 3 FOLLOWER]: Leader pre-election won for term 4
10:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.151492 11680 raft_consensus.cc:491] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 585db50113304f22ac16c7e112fcddab)
10:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.151571 11680 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 3 FOLLOWER]: Advancing to term 4
10:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.157842 11461 leader_election.cc:336] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.161669 11641 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d989751ff71c4a4b81b2f08770595162" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" is_pre_election: true
10:05:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.162822 11680 raft_consensus.cc:513] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.163342 11641 raft_consensus.cc:2388] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 4 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d989751ff71c4a4b81b2f08770595162 in current term 4: Already voted for candidate f4e788cbdfb84919a6ecb74d02dbd0b0 in this term.
10:05:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.163928 11680 leader_election.cc:290] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 4 election: Requested vote from peers d989751ff71c4a4b81b2f08770595162 (127.10.85.190:43483), 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575)
10:05:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.164753 11488 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "d989751ff71c4a4b81b2f08770595162"
10:05:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.164916 11488 raft_consensus.cc:3054] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 3 FOLLOWER]: Advancing to term 4
10:05:38.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823; will retry
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.167075 11615 leader_election.cc:336] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.167239 11615 leader_election.cc:336] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111)
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.167392 11488 raft_consensus.cc:2463] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f4e788cbdfb84919a6ecb74d02dbd0b0 in term 4.
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.168268 11617 leader_election.cc:304] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d989751ff71c4a4b81b2f08770595162, f4e788cbdfb84919a6ecb74d02dbd0b0; no voters: 585db50113304f22ac16c7e112fcddab
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.168342 11463 leader_election.cc:304] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d989751ff71c4a4b81b2f08770595162; no voters: 585db50113304f22ac16c7e112fcddab, f4e788cbdfb84919a6ecb74d02dbd0b0
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.168710 11680 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 4 FOLLOWER]: Leader election won for term 4
10:05:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.168704 11703 raft_consensus.cc:2743] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 4 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
10:05:38.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.168844 11680 raft_consensus.cc:695] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [term 4 LEADER]: Becoming Leader. State: Replica: f4e788cbdfb84919a6ecb74d02dbd0b0, State: Running, Role: LEADER
10:05:38.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.169229 11680 consensus_queue.cc:237] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } }
10:05:38.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.170440 11702 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4e788cbdfb84919a6ecb74d02dbd0b0. Latest consensus state: current_term: 4 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:38.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.170626 11702 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: LEADER
10:05:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.171232 11708 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:05:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.173436 11708 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d]
10:05:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.173951 11708 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243]
10:05:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.175307 11708 tablet_loader.cc:96] loaded metadata for tablet 03f537e9861247ecaa40c36fce6bcd91 (table kudu_system.kudu_transactions [id=712ca2b4bf2f45e19a7d94a82839fc6d])
10:05:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.175832 11708 tablet_loader.cc:96] loaded metadata for tablet 59ba63bafba443cdb09fba9d4814185f (table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243])
10:05:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.176204 11708 tablet_loader.cc:96] loaded metadata for tablet 6ae46b0d157a4cd98d06940e64ffe5af (table txn_manager_fallback_rolling [id=8dd388c02673449b9d85313eaa39f243])
10:05:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.176539 11708 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:05:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.177261 11708 catalog_manager.cc:1260] Loaded cluster ID: 972bcc6920f5498291b9d099a339aefc
10:05:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.177354 11708 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:05:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.178414 11708 catalog_manager.cc:1505] Loading token signing keys...
10:05:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.179175 11708 catalog_manager.cc:5910] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0: Loaded TSK: 0
10:05:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.179916 11708 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:05:38.192 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.85.189:33575(127.10.85.189:33575)] connecting to peer
10:05:38.193 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.189:33575(127.10.85.189:33575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.85.189:33575: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575
10:05:38.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:05:38.194 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=101), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [75ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [76ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, deferred=Deferred@2041227300(state=PENDING, result=null, callback=(continuation of Deferred@1533446658 after retry RPC after error@1505825688) -> (continuation of Deferred@946520728 after retry RPC after error@346928697) -> (continuation of Deferred@2071201844 after retry RPC after error@712542481) -> (continuation of Deferred@1156116676 after retry RPC after error@1024297502), errback=(continuation of Deferred@1533446658 after retry RPC after error@1505825688) -> (continuation of Deferred@946520728 after retry RPC after error@346928697) -> (continuation of Deferred@2071201844 after retry RPC after error@712542481) -> (continuation of Deferred@1156116676 after retry RPC after error@1024297502))) after lookup
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.258035 11631 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47112:
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:05:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:05:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.260941 11676 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
10:05:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.274022 11615 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Couldn't send request to peer 585db50113304f22ac16c7e112fcddab. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:05:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.279022 11488 raft_consensus.cc:1270] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [term 4 FOLLOWER]: Refusing update from remote peer f4e788cbdfb84919a6ecb74d02dbd0b0: Log matching property violated. Preceding OpId in replica: term: 3 index: 13. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
10:05:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.279700 11702 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
10:05:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.281340 11703 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4e788cbdfb84919a6ecb74d02dbd0b0. Latest consensus state: current_term: 4 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.281661 11703 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.282666 11702 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.282895 11702 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 [sys.catalog]: This master's current role is: LEADER
10:05:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.286480 11703 sys_catalog.cc:455] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d989751ff71c4a4b81b2f08770595162" member_type: VOTER last_known_addr { host: "127.10.85.190" port: 43483 } } peers { permanent_uuid: "585db50113304f22ac16c7e112fcddab" member_type: VOTER last_known_addr { host: "127.10.85.189" port: 33575 } } peers { permanent_uuid: "f4e788cbdfb84919a6ecb74d02dbd0b0" member_type: VOTER last_known_addr { host: "127.10.85.188" port: 35823 } } }
10:05:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.286726 11703 sys_catalog.cc:458] T 00000000000000000000000000000000 P d989751ff71c4a4b81b2f08770595162 [sys.catalog]: This master's current role is: FOLLOWER
10:05:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.343272 11661 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47104 (request call id 11) took 148 ms (client timeout 148 ms). Trace:
10:05:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.343468 11314 heartbeater.cc:346] Connected to a master server at 127.10.85.188:35823
10:05:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.343426 11661 rpcz_store.cc:269] 0115 10:05:38.195028 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:05:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.195086 (+ 58us) service_pool.cc:225] Handling call
10:05:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.343215 (+148129us) inbound_call.cc:173] Queueing success response
10:05:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":571,"init.queue_time_us":43,"thread_start_us":1327,"threads_started":4}
10:05:38.344 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [75ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [76ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [102ms] sending RPC to server master-127.10.85.188:35823, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2041227300(state=PENDING, result=null, callback=(continuation of Deferred@1533446658 after retry RPC after error@1505825688) -> (continuation of Deferred@946520728 after retry RPC after error@346928697) -> (continuation of Deferred@2071201844 after retry RPC after error@712542481) -> (continuation of Deferred@1156116676 after retry RPC after error@1024297502) -> (continuation of Deferred@1726156835 after retry RPC@138231279), errback=(continuation of Deferred@1533446658 after retry RPC after error@1505825688) -> (continuation of Deferred@946520728 after retry RPC after error@346928697) -> (continuation of Deferred@2071201844 after retry RPC after error@712542481) -> (continuation of Deferred@1156116676 after retry RPC after error@1024297502) -> (continuation of Deferred@1726156835 after retry RPC@138231279)))
10:05:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.344560 11631 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "cb958ffb8ba54460a13472fe93aba131" instance_seqno: 1736935531991487) as {username='slave'} at 127.10.85.131:36821; Asking this server to re-register.
10:05:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.345031 11314 heartbeater.cc:463] Registering TS with master...
10:05:38.345 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [20ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [75ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [76ms] delaying RPC due to: Service unavailable: Master config (127.10.85.189:33575,127.10.85.190:43483,127.10.85.188:35823) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.189:33575, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.190:43483, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.85.188:35823, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.189:33575: Network error: Failed to connect to peer master-127.10.85.189:33575(127.10.85.189:33575): Connection refused: /127.10.85.189:33575, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.188:35823: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.85.190:43483: OK, [102ms] sending RPC to server master-127.10.85.188:35823, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2041227300(state=PENDING, result=null, callback=(continuation of Deferred@1533446658 after retry RPC after error@1505825688) -> (continuation of Deferred@946520728 after retry RPC after error@346928697) -> (continuation of Deferred@2071201844 after retry RPC after error@712542481) -> (continuation of Deferred@1156116676 after retry RPC after error@1024297502) -> (continuation of Deferred@1726156835 after retry RPC@138231279), errback=(continuation of Deferred@1533446658 after retry RPC after error@1505825688) -> (continuation of Deferred@946520728 after retry RPC after error@346928697) -> (continuation of Deferred@2071201844 after retry RPC after error@712542481) -> (continuation of Deferred@1156116676 after retry RPC after error@1024297502) -> (continuation of Deferred@1726156835 after retry RPC@138231279)))
10:05:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.345243 11314 heartbeater.cc:509] Master 127.10.85.188:35823 requested a full tablet report, sending...
10:05:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.346217 11631 ts_manager.cc:194] Registered new tserver with Master: cb958ffb8ba54460a13472fe93aba131 (127.10.85.131:32801)
10:05:38.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.375561 11172 heartbeater.cc:346] Connected to a master server at 127.10.85.188:35823
10:05:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.376574 11631 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "34c6f1b7300a46919cb1515f6f721cbc" instance_seqno: 1736935531556451) as {username='slave'} at 127.10.85.130:38117; Asking this server to re-register.
10:05:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.376998 11172 heartbeater.cc:463] Registering TS with master...
10:05:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.377241 11172 heartbeater.cc:509] Master 127.10.85.188:35823 requested a full tablet report, sending...
10:05:38.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.378283 11631 ts_manager.cc:194] Registered new tserver with Master: 34c6f1b7300a46919cb1515f6f721cbc (127.10.85.130:45325)
10:05:38.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:38.570128 10914 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.186s from last keepalive heartbeat (effective timeout is 1.000s)
10:05:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.576429 10949 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.10.85.190:43483)
10:05:38.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.591776 11726 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
10:05:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.592255 11725 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
10:05:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.603034 11661 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47104 (request call id 12) took 250 ms (client timeout 250 ms). Trace:
10:05:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.603181 11661 rpcz_store.cc:269] 0115 10:05:38.352821 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:05:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.352874 (+ 53us) service_pool.cc:225] Handling call
10:05:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.602958 (+250084us) inbound_call.cc:173] Queueing success response
10:05:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:05:38.604 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@198214396(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@5adcb4e2, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@2440c592))
10:05:38.604 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@198214396(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@5adcb4e2, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@2440c592))
10:05:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.747514 11615 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Couldn't send request to peer 585db50113304f22ac16c7e112fcddab. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:05:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.862586 11661 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47104 (request call id 13) took 250 ms (client timeout 250 ms). Trace:
10:05:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:38.862751 11661 rpcz_store.cc:269] 0115 10:05:38.612396 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:05:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.612483 (+ 87us) service_pool.cc:225] Handling call
10:05:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.862535 (+250052us) inbound_call.cc:173] Queueing success response
10:05:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:05:38.863 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1165186736(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@4abbeb80, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@75ff2d91))
10:05:38.863 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1165186736(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@4abbeb80, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@75ff2d91))
10:05:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:39.123135 11661 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47104 (request call id 14) took 250 ms (client timeout 250 ms). Trace:
10:05:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:39.123247 11661 rpcz_store.cc:269] 0115 10:05:38.872960 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:05:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:38.873023 (+ 63us) service_pool.cc:225] Handling call
10:05:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:05:39.123076 (+250053us) inbound_call.cc:173] Queueing success response
10:05:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:05:39.124 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1471220457(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@8280079, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@1dda1d17))
10:05:39.124 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.10.85.188:35823, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1471220457(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@8280079, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@1dda1d17))
10:05:39.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:39.244542 11615 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Couldn't send request to peer 585db50113304f22ac16c7e112fcddab. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:39.274688 11630 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47124:
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:05:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:05:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:05:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:05:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:05:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:39.275379 11630 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:05:39.288 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1736935532 last_transition_timestamp: 1736935538
10:05:39.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:39.776046 11615 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Couldn't send request to peer 585db50113304f22ac16c7e112fcddab. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:05:39.879 [DEBUG - main] (Connection.java:575) [peer master-127.10.85.188:35823(127.10.85.188:35823)] disconnecting while in state READY
10:05:39.879 [DEBUG - main] (Connection.java:575) [peer master-127.10.85.190:43483(127.10.85.190:43483)] disconnecting while in state READY
10:05:39.879 [DEBUG - main] (Connection.java:575) [peer 34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325)] disconnecting while in state READY
10:05:39.879 [DEBUG - main] (Connection.java:575) [peer cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)] disconnecting while in state READY
10:05:39.879 [DEBUG - main] (Connection.java:575) [peer master-127.10.85.189:33575(127.10.85.189:33575)] disconnecting while in state TERMINATED
10:05:39.880 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.85.188:35823(127.10.85.188:35823)] handling channelInactive
10:05:39.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.85.188:35823(127.10.85.188:35823)] cleaning up while in state READY due to: connection closed
10:05:39.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:05:39.881 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.85.190:43483(127.10.85.190:43483)] handling channelInactive
10:05:39.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.85.190:43483(127.10.85.190:43483)] cleaning up while in state READY due to: connection closed
10:05:39.881 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325)] handling channelInactive
10:05:39.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 34c6f1b7300a46919cb1515f6f721cbc(127.10.85.130:45325)] cleaning up while in state READY due to: connection closed
10:05:39.881 [DEBUG - kudu-nio-1] (Connection.java:239) [peer cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)] handling channelInactive
10:05:39.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cb958ffb8ba54460a13472fe93aba131(127.10.85.131:32801)] cleaning up while in state READY due to: connection closed
10:05:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:39.881721 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 10874
10:05:39.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:39.887295 11032 heartbeater.cc:501] Master 127.10.85.188:35823 was elected leader, sending a full tablet report...
10:05:39.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:39.893433 11027 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:40.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.005522 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 10874
10:05:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.026510 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 11037
10:05:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:40.028514 11053 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.129:43803: connect: Connection refused (error 111)
10:05:40.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:40.029810 11053 consensus_peers.cc:487] T 6ae46b0d157a4cd98d06940e64ffe5af P 34c6f1b7300a46919cb1515f6f721cbc -> Peer 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803): Couldn't send request to peer 4a5806654c4148fbb6e787f4ad65533a. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.129:43803: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:05:40.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.033070 11167 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:40.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.122953 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 11037
10:05:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:40.133713 11194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.85.129:43803: connect: Connection refused (error 111)
10:05:40.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:40.134930 11194 consensus_peers.cc:487] T 59ba63bafba443cdb09fba9d4814185f P cb958ffb8ba54460a13472fe93aba131 -> Peer 4a5806654c4148fbb6e787f4ad65533a (127.10.85.129:43803): Couldn't send request to peer 4a5806654c4148fbb6e787f4ad65533a. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.129:43803: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:05:40.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:40.140358 11193 connection.cc:537] server connection from 127.10.85.130:60741 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:05:40.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.140858 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 11177
10:05:40.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.151252 11309 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.240720 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 11177
10:05:40.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.248220 11445 master_runner.cc:186] Error getting master registration for 127.10.85.189:33575: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111),
10:05:40.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.248440 11445 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:05:40.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:05:40.257054 11615 consensus_peers.cc:487] T 00000000000000000000000000000000 P f4e788cbdfb84919a6ecb74d02dbd0b0 -> Peer 585db50113304f22ac16c7e112fcddab (127.10.85.189:33575): Couldn't send request to peer 585db50113304f22ac16c7e112fcddab. Status: Network error: Client connection negotiation failed: client connection to 127.10.85.189:33575: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:05:40.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.257946 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 11445
10:05:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.263615 11518 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.363890 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 11445
10:05:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.383168 10582 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu pid 11598
10:05:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.388425 11671 generic_service.cc:196] Checking for leaks (request via RPC)
10:05:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:05:40.489737 10582 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskkcvsP0/build/asan/bin/kudu with pid 11598
10:05:40.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:05:40Z chronyd exiting
10:05:40.630 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)