10:08:52.173 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
10:08:52.173 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskblP3Kh/test-tmp
10:08:52.173 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
10:08:52.173 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskblP3Kh/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
10:08:52.174 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
10:08:52.482 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:08:52.482 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:08:52.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.484591 13109 mini_oidc.cc:152] Starting JWKS server
10:08:52.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.487062 13109 webserver.cc:460] Webserver started at http://127.0.0.1:34589/ using document root <none> and password file <none>
10:08:52.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.487260 13109 mini_oidc.cc:189] Starting OIDC Discovery server
10:08:52.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.487767 13109 webserver.cc:460] Webserver started at http://127.0.0.1:37863/ using document root <none> and password file <none>
10:08:52.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T10:08:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
10:08:52.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T10:08:52Z Disabled control of system clock
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.516206 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/logs
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/info.pb
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.126:37355
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.126
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:08:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:08:52.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.825210 13120 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:52.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.825515 13120 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:52.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.825604 13120 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833218 13120 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833328 13120 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833364 13120 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833413 13120 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833453 13120 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833501 13120 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:08:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.833532 13120 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34589/jwks.json
10:08:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.842730 13120 master_runner.cc:386] Master server non-default flags:
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.126:37355
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/info.pb
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.126
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/logs
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:52.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:52.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.843967 13120 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:52.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.845697 13120 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.855806 13126 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.856184 13125 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.856562 13128 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.857021 13120 server_base.cc:1034] running on GCE node
10:08:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.857681 13120 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:52.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.858989 13120 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.860178 13120 hybrid_clock.cc:648] HybridClock initialized: now 1714126132860134 us; error 36 us; skew 500 ppm
10:08:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.860544 13120 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.862624 13120 webserver.cc:460] Webserver started at http://127.12.205.126:41957/ using document root <none> and password file <none>
10:08:52.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.863289 13120 fs_manager.cc:359] Metadata directory not provided
10:08:52.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.863400 13120 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:52.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.863691 13120 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:08:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.866147 13120 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/instance:
10:08:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a63e2d0f1f62473395e18c56e6b648bd"
10:08:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:52 on dist-test-slave-ww78"
10:08:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.866923 13120 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal/instance:
10:08:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a63e2d0f1f62473395e18c56e6b648bd"
10:08:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:52 on dist-test-slave-ww78"
10:08:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.871745 13120 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
10:08:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.874930 13134 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.876142 13120 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.876298 13120 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a63e2d0f1f62473395e18c56e6b648bd"
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:52 on dist-test-slave-ww78"
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.876544 13120 fs_report.cc:383] FS layout report
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/data
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:52.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.898488 13120 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.899333 13120 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.899611 13120 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.918895 13120 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.126:37355
10:08:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.918918 13195 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.126:37355 every 8 connection(s)
10:08:52.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.920223 13120 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/info.pb
10:08:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.921581 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13120
10:08:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.921831 13109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal/instance
10:08:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.923797 13196 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.923995 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/logs
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/info.pb
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.125:45397
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.125
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:08:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:08:52.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.929304 13196 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } has no permanent_uuid. Determining permanent_uuid...
10:08:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.943884 13196 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } has no permanent_uuid. Determining permanent_uuid...
10:08:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.944903 13135 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:52.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.946322 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.125:45397: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:52.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:52.995620 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } attempt: 1
10:08:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:52.997596 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.125:45397: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:53.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.065840 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } attempt: 2
10:08:53.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.067735 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.125:45397: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:53.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.158991 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } attempt: 3
10:08:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.160920 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.125:45397: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.229522 13198 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.229815 13198 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.229907 13198 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237112 13198 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237213 13198 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237251 13198 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237299 13198 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237337 13198 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237387 13198 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:08:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.237427 13198 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34589/jwks.json
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.246235 13198 master_runner.cc:386] Master server non-default flags:
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.125:45397
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/info.pb
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.125
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/logs
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.247471 13198 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.249132 13198 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.258885 13206 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.259150 13207 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.259539 13209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.259974 13198 server_base.cc:1034] running on GCE node
10:08:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.260663 13198 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.261977 13198 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.263164 13198 hybrid_clock.cc:648] HybridClock initialized: now 1714126133263110 us; error 52 us; skew 500 ppm
10:08:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.263468 13198 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.265534 13198 webserver.cc:460] Webserver started at http://127.12.205.125:36539/ using document root <none> and password file <none>
10:08:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.266192 13198 fs_manager.cc:359] Metadata directory not provided
10:08:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.266274 13198 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.266515 13198 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:08:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.269028 13198 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/instance:
10:08:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42436150052e4f878ad40cb5c962c74f"
10:08:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:08:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.269747 13198 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal/instance:
10:08:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42436150052e4f878ad40cb5c962c74f"
10:08:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:08:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.274363 13198 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
10:08:53.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.277585 13215 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:53.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.278791 13198 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.004s sys 0.000s
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.278929 13198 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42436150052e4f878ad40cb5c962c74f"
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.279088 13198 fs_report.cc:383] FS layout report
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/data
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:53.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.296006 13198 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:53.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.296792 13198 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:53.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.297173 13198 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.304159 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } attempt: 4
10:08:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.306277 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.125:45397: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.316383 13198 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.125:45397
10:08:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.316390 13276 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.125:45397 every 8 connection(s)
10:08:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.317705 13198 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/info.pb
10:08:53.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.318017 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13198
10:08:53.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.318154 13109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal/instance
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.319394 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/logs
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data/info.pb
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.124:46319
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.124
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:08:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:08:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.322072 13277 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.327740 13277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.341073 13277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.349843 13277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.350780 13218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.352180 13277 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.401455 13277 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 1
10:08:53.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.403219 13277 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.471411 13277 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 2
10:08:53.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.473219 13277 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.564462 13277 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 3
10:08:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.566326 13277 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.605522 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } attempt: 5
10:08:53.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.611685 13196 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.613413 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.629503 13279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.629776 13279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.629861 13279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637018 13279 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637115 13279 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637151 13279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637199 13279 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637236 13279 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637295 13279 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:08:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.637334 13279 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34589/jwks.json
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.646399 13279 master_runner.cc:386] Master server non-default flags:
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.124:46319
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data/info.pb
10:08:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.124
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/logs
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.647634 13279 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.649273 13279 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.659356 13287 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.659670 13290 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:53.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.660049 13288 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:53.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.660526 13279 server_base.cc:1034] running on GCE node
10:08:53.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.661180 13279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.662477 13279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:53.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.663677 13279 hybrid_clock.cc:648] HybridClock initialized: now 1714126133663629 us; error 39 us; skew 500 ppm
10:08:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.664026 13279 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.664587 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 1
10:08:53.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.666309 13196 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.666334 13279 webserver.cc:460] Webserver started at http://127.12.205.124:42833/ using document root <none> and password file <none>
10:08:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.666955 13279 fs_manager.cc:359] Metadata directory not provided
10:08:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.667037 13279 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.667294 13279 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:08:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.669721 13279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data/instance:
10:08:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "051c8147615a410d851ef3d6db946e03"
10:08:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:08:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.670416 13279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal/instance:
10:08:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "051c8147615a410d851ef3d6db946e03"
10:08:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:08:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.675006 13279 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
10:08:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.678041 13296 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.679196 13279 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.679335 13279 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "051c8147615a410d851ef3d6db946e03"
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.679488 13279 fs_report.cc:383] FS layout report
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data/data
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.705910 13279 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.706722 13279 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.706971 13279 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.709514 13277 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 4
10:08:53.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.711313 13277 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:08:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.726310 13279 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.124:46319
10:08:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.726330 13357 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.124:46319 every 8 connection(s)
10:08:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.727520 13279 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/data/info.pb
10:08:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.730979 13358 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.733876 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13279
10:08:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.734021 13109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-2/wal/instance
10:08:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.734527 13196 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 2
10:08:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.737497 13358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.750352 13358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.756541 13196 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Bootstrap starting.
10:08:53.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.758093 13358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } has no permanent_uuid. Determining permanent_uuid...
10:08:53.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.760128 13196 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Neither blocks nor log segments found. Creating new log.
10:08:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.761385 13196 log.cc:824] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Log is configured to *not* fsync() on all Append() calls
10:08:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.766448 13196 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: No bootstrap required, opened a new log
10:08:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.767776 13358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: Bootstrap starting.
10:08:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.770704 13358 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: Neither blocks nor log segments found. Creating new log.
10:08:53.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.771726 13358 log.cc:824] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: Log is configured to *not* fsync() on all Append() calls
10:08:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.773737 13196 raft_consensus.cc:348] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.773998 13196 raft_consensus.cc:374] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.774073 13196 raft_consensus.cc:729] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a63e2d0f1f62473395e18c56e6b648bd, State: Initialized, Role: FOLLOWER
10:08:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.774222 13358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: No bootstrap required, opened a new log
10:08:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.774667 13196 consensus_queue.cc:260] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.776294 13368 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.776628 13368 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:08:53.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.777356 13196 sys_catalog.cc:564] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: configured and running, proceeding with master startup.
10:08:53.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.780920 13358 raft_consensus.cc:348] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.781235 13358 raft_consensus.cc:374] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:53.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.781349 13358 raft_consensus.cc:729] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 051c8147615a410d851ef3d6db946e03, State: Initialized, Role: FOLLOWER
10:08:53.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.781999 13358 consensus_queue.cc:260] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.783344 13368 raft_consensus.cc:482] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:08:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.783376 13379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.783561 13368 raft_consensus.cc:504] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.783726 13379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: FOLLOWER
10:08:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.784392 13358 sys_catalog.cc:564] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: configured and running, proceeding with master startup.
10:08:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.785141 13368 leader_election.cc:288] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:08:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.786451 13323 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a63e2d0f1f62473395e18c56e6b648bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "051c8147615a410d851ef3d6db946e03" is_pre_election: true
10:08:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.786958 13323 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a63e2d0f1f62473395e18c56e6b648bd in term 0.
10:08:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.788034 13137 leader_election.cc:302] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 051c8147615a410d851ef3d6db946e03, a63e2d0f1f62473395e18c56e6b648bd; no voters:
10:08:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.788903 13368 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Leader pre-election won for term 1
10:08:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.789059 13368 raft_consensus.cc:482] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:08:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.789175 13368 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 0 FOLLOWER]: Advancing to term 1
10:08:53.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.792222 13368 raft_consensus.cc:504] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.795150 13384 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:08:53.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.795264 13384 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:08:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.797068 13323 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a63e2d0f1f62473395e18c56e6b648bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "051c8147615a410d851ef3d6db946e03"
10:08:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.797299 13323 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 0 FOLLOWER]: Advancing to term 1
10:08:53.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.806136 13323 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a63e2d0f1f62473395e18c56e6b648bd in term 1.
10:08:53.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.806643 13368 leader_election.cc:288] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 1 election: Requested vote from peers 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:08:53.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.807830 13137 leader_election.cc:302] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 051c8147615a410d851ef3d6db946e03, a63e2d0f1f62473395e18c56e6b648bd; no voters:
10:08:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.797614 13120 master_runner.cc:186] Error getting master registration for 127.12.205.125:45397: OK, instance_id { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" instance_seqno: 1714126133311889 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:08:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.810606 13120 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:08:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.812611 13368 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Leader election won for term 1
10:08:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.807745 13279 master_runner.cc:186] Error getting master registration for 127.12.205.125:45397: OK, instance_id { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" instance_seqno: 1714126133311889 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:08:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.817364 13279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:08:53.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.819110 13392 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:08:53.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:53.819351 13392 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:08:53.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.820770 13368 raft_consensus.cc:686] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 LEADER]: Becoming Leader. State: Replica: a63e2d0f1f62473395e18c56e6b648bd, State: Running, Role: LEADER
10:08:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.821482 13368 consensus_queue.cc:237] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.824697 13394 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: New leader a63e2d0f1f62473395e18c56e6b648bd. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.825076 13394 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: LEADER
10:08:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.825932 13396 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
10:08:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.828276 13396 catalog_manager.cc:1484] Initializing Kudu cluster ID...
10:08:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.837944 13323 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 1 FOLLOWER]: Refusing update from remote peer a63e2d0f1f62473395e18c56e6b648bd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:08:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.838862 13394 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.858701 13379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a63e2d0f1f62473395e18c56e6b648bd. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.858929 13379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: FOLLOWER
10:08:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.860680 13397 mvcc.cc:205] Tried to move back new op lower bound from 7021060644189392896 to 7021060644137840640. Current Snapshot: MvccSnapshot[applied={T|T < 7021060644189392896}]
10:08:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.864439 13368 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.864840 13368 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: LEADER
10:08:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.867074 13396 catalog_manager.cc:1347] Generated new cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:08:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.867173 13396 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
10:08:53.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.876952 13379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:53.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.877208 13379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: FOLLOWER
10:08:53.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.893292 13396 catalog_manager.cc:1370] Generated new certificate authority record
10:08:53.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.894937 13396 catalog_manager.cc:1504] Loading token signing keys...
10:08:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.910662 13396 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Generated new TSK 0
10:08:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.912968 13396 catalog_manager.cc:1514] Initializing in-progress tserver states...
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:53.929059 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/logs
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data/info.pb
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.65:0
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.205.65
10:08:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.65
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:08:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:08:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.010533 13277 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } attempt: 5
10:08:54.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.023839 13277 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Bootstrap starting.
10:08:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.027913 13277 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Neither blocks nor log segments found. Creating new log.
10:08:54.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.030506 13277 log.cc:824] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Log is configured to *not* fsync() on all Append() calls
10:08:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.044855 13277 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: No bootstrap required, opened a new log
10:08:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.061923 13277 raft_consensus.cc:348] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.062791 13277 raft_consensus.cc:374] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.063148 13277 raft_consensus.cc:729] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42436150052e4f878ad40cb5c962c74f, State: Initialized, Role: FOLLOWER
10:08:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.064808 13277 consensus_queue.cc:260] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.077003 13404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.077423 13404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: FOLLOWER
10:08:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.080659 13241 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a63e2d0f1f62473395e18c56e6b648bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42436150052e4f878ad40cb5c962c74f"
10:08:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.081112 13242 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a63e2d0f1f62473395e18c56e6b648bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42436150052e4f878ad40cb5c962c74f" is_pre_election: true
10:08:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.082262 13240 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 0 FOLLOWER]: Advancing to term 1
10:08:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.085860 13242 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59084 (request call id 1) took 300 ms (client timeout 381 ms). Trace:
10:08:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.085985 13242 rpcz_store.cc:269] 0426 10:08:53.785480 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:08:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:53.785786 (+ 306us) service_pool.cc:225] Handling call
10:08:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:54.085465 (+299679us) spinlock_profiling.cc:243] Waited 3.46 ms on lock 0x61800000e880. stack: 00007f4892833356 00007f4892833110 00007f489186723d 0000000000954e64 00007f48925d0a7a 00007f48925e3914 00007f48925dfc29 00007f489e08568b 00007f48a1d50f84 00007f489c255996 00007f489c2556c5 00007f48995364fe 00007f48995353de 00007f489953b8b5 00007f489953c1df 00007f489953c008
10:08:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:54.085810 (+ 345us) inbound_call.cc:173] Queueing success response
10:08:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":7950720}
10:08:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.088487 13241 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59084 (request call id 3) took 290 ms (client timeout 340 ms). Trace:
10:08:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.088594 13241 rpcz_store.cc:269] 0426 10:08:53.797533 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:08:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:53.797588 (+ 55us) service_pool.cc:225] Handling call
10:08:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:54.088324 (+290736us) spinlock_profiling.cc:243] Waited 7.22 ms on lock 0x61800000e880. stack: 00007f4892833356 00007f4892833110 00007f489186723d 0000000000954e64 00007f48925d0a7a 00007f48925e3914 00007f48925dfc29 00007f489e08568b 00007f48a1d50f84 00007f489c255996 00007f489c2556c5 00007f48995364fe 00007f48995353de 00007f489953b8b5 00007f489953c1df 00007f489953c008
10:08:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:54.088451 (+ 127us) inbound_call.cc:173] Queueing success response
10:08:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":16600448}
10:08:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.091298 13240 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 1 FOLLOWER]: Refusing update from remote peer a63e2d0f1f62473395e18c56e6b648bd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:08:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.092120 13394 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.098616 13277 sys_catalog.cc:564] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: configured and running, proceeding with master startup.
10:08:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.144263 13404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: New leader a63e2d0f1f62473395e18c56e6b648bd. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.144578 13404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: FOLLOWER
10:08:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.145056 13411 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.145537 13411 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: FOLLOWER
10:08:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.173655 13420 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:08:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.178197 13394 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a63e2d0f1f62473395e18c56e6b648bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.178485 13394 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: LEADER
10:08:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.355859 13402 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.356148 13402 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.356307 13402 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.363641 13402 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.363803 13402 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:08:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.363862 13402 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:08:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.363924 13402 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:08:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.364027 13402 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.205.65
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.372884 13402 tablet_server_runner.cc:78] Tablet server non-default flags:
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.65:0
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data/info.pb
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.65
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.205.65
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/logs
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:54.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.374457 13402 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:54.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.376724 13402 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:54.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.390691 13425 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:54.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.392334 13426 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:54.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.393545 13428 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.393944 13402 server_base.cc:1034] running on GCE node
10:08:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.394619 13402 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.395721 13402 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.396939 13402 hybrid_clock.cc:648] HybridClock initialized: now 1714126134396942 us; error 96 us; skew 500 ppm
10:08:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.397287 13402 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:54.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.399718 13402 webserver.cc:460] Webserver started at http://127.12.205.65:44393/ using document root <none> and password file <none>
10:08:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.400445 13402 fs_manager.cc:359] Metadata directory not provided
10:08:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.400535 13402 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.400781 13402 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:08:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.403177 13402 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data/instance:
10:08:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e999584f2f6417c9a44626007db7e78"
10:08:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:54 on dist-test-slave-ww78"
10:08:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.403919 13402 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal/instance:
10:08:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e999584f2f6417c9a44626007db7e78"
10:08:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:54 on dist-test-slave-ww78"
10:08:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.408833 13402 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.003s
10:08:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.411971 13434 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.413254 13402 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:08:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.413409 13402 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal
10:08:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e999584f2f6417c9a44626007db7e78"
10:08:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:54 on dist-test-slave-ww78"
10:08:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.413597 13402 fs_report.cc:383] FS layout report
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data/data
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:54.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.429806 13402 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:54.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.430632 13402 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.430912 13402 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.433812 13402 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:08:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.433912 13402 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:08:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.434020 13402 ts_tablet_manager.cc:590] Registered 0 tablets
10:08:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.434077 13402 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:08:54.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.486888 13402 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.65:43323
10:08:54.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.486959 13556 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.65:43323 every 8 connection(s)
10:08:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.488076 13402 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/data/info.pb
10:08:54.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.491919 13558 heartbeater.cc:347] Connected to a master server at 127.12.205.126:37355
10:08:54.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.492174 13558 heartbeater.cc:464] Registering TS with master...
10:08:54.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.493523 13558 heartbeater.cc:511] Master 127.12.205.126:37355 requested a full tablet report, sending...
10:08:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.495915 13151 ts_manager.cc:194] Registered new tserver with Master: 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323)
10:08:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.497114 13557 heartbeater.cc:347] Connected to a master server at 127.12.205.124:46319
10:08:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.497241 13557 heartbeater.cc:464] Registering TS with master...
10:08:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.497929 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13402
10:08:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.497905 13151 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.205.65:51009
10:08:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.498088 13109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-0/wal/instance
10:08:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.499605 13557 heartbeater.cc:511] Master 127.12.205.124:46319 requested a full tablet report, sending...
10:08:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.501238 13312 ts_manager.cc:194] Registered new tserver with Master: 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323)
10:08:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.502679 13559 heartbeater.cc:347] Connected to a master server at 127.12.205.125:45397
10:08:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.502815 13559 heartbeater.cc:464] Registering TS with master...
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.503158 13559 heartbeater.cc:511] Master 127.12.205.125:45397 requested a full tablet report, sending...
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.503229 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/logs
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data/info.pb
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.66:0
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.205.66
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.66
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:08:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.504786 13232 ts_manager.cc:194] Registered new tserver with Master: 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323)
10:08:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.816083 13562 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.816354 13562 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.816499 13562 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.821310 13392 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:08:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.821403 13392 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: loading cluster ID for follower catalog manager: success
10:08:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.823900 13392 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: acquiring CA information for follower catalog manager: success
10:08:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.823920 13562 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.824085 13562 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:08:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.824141 13562 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:08:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.824205 13562 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:08:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.824314 13562 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.205.66
10:08:54.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.825666 13392 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.833482 13562 tablet_server_runner.cc:78] Tablet server non-default flags:
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.66:0
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data/info.pb
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.66
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.205.66
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/logs
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.835050 13562 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.837080 13562 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.848922 13568 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.849284 13567 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:54.849591 13570 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.850133 13562 server_base.cc:1034] running on GCE node
10:08:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.850678 13562 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.851542 13562 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.852715 13562 hybrid_clock.cc:648] HybridClock initialized: now 1714126134852664 us; error 51 us; skew 500 ppm
10:08:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.852996 13562 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.855154 13562 webserver.cc:460] Webserver started at http://127.12.205.66:44963/ using document root <none> and password file <none>
10:08:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.855892 13562 fs_manager.cc:359] Metadata directory not provided
10:08:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.855980 13562 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.856245 13562 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:08:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.858829 13562 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data/instance:
10:08:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d61405dbc7b04c15a334468216058bc1"
10:08:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:54 on dist-test-slave-ww78"
10:08:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.859515 13562 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal/instance:
10:08:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d61405dbc7b04c15a334468216058bc1"
10:08:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:54 on dist-test-slave-ww78"
10:08:54.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.864320 13562 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.001s
10:08:54.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.867569 13576 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:54.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.868878 13562 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.869010 13562 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d61405dbc7b04c15a334468216058bc1"
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:54 on dist-test-slave-ww78"
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.869172 13562 fs_report.cc:383] FS layout report
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data/data
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:54.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.908448 13562 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.909300 13562 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.909543 13562 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.914688 13562 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:08:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.914794 13562 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:08:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.914932 13562 ts_tablet_manager.cc:590] Registered 0 tablets
10:08:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.914999 13562 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:08:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.971520 13562 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.66:37261
10:08:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.971554 13697 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.66:37261 every 8 connection(s)
10:08:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.972776 13562 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/data/info.pb
10:08:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.976222 13699 heartbeater.cc:347] Connected to a master server at 127.12.205.126:37355
10:08:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.976493 13699 heartbeater.cc:464] Registering TS with master...
10:08:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.977474 13699 heartbeater.cc:511] Master 127.12.205.126:37355 requested a full tablet report, sending...
10:08:54.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.979595 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13562
10:08:54.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.979759 13109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-1/wal/instance
10:08:54.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.981438 13151 ts_manager.cc:194] Registered new tserver with Master: d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:54.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.982517 13151 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.205.66:44779
10:08:54.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.983563 13698 heartbeater.cc:347] Connected to a master server at 127.12.205.124:46319
10:08:54.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.983709 13698 heartbeater.cc:464] Registering TS with master...
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.984246 13698 heartbeater.cc:511] Master 127.12.205.124:46319 requested a full tablet report, sending...
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.984418 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/logs
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data/info.pb
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.67:0
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.205.67
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.67
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
10:08:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.985821 13312 ts_manager.cc:194] Registered new tserver with Master: d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:54.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.988109 13700 heartbeater.cc:347] Connected to a master server at 127.12.205.125:45397
10:08:54.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.988229 13700 heartbeater.cc:464] Registering TS with master...
10:08:54.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.988526 13700 heartbeater.cc:511] Master 127.12.205.125:45397 requested a full tablet report, sending...
10:08:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:54.989531 13232 ts_manager.cc:194] Registered new tserver with Master: d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:55.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.178743 13420 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:08:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.178880 13420 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: loading cluster ID for follower catalog manager: success
10:08:55.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.182897 13420 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: acquiring CA information for follower catalog manager: success
10:08:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.185961 13420 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:08:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.303870 13703 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.304170 13703 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.304332 13703 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.312373 13703 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.312584 13703 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
10:08:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.312638 13703 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
10:08:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.312705 13703 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
10:08:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.312808 13703 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.205.67
10:08:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.322690 13703 tablet_server_runner.cc:78] Tablet server non-default flags:
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.67:0
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data/info.pb
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.67
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.205.67
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/logs
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:55.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.324237 13703 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:55.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.326040 13703 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.338379 13710 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.338593 13709 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:55.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.339442 13712 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:55.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.340037 13703 server_base.cc:1034] running on GCE node
10:08:55.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.340790 13703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:55.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.341899 13703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:55.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.343125 13703 hybrid_clock.cc:648] HybridClock initialized: now 1714126135343073 us; error 43 us; skew 500 ppm
10:08:55.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.343501 13703 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:55.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.346159 13703 webserver.cc:460] Webserver started at http://127.12.205.67:40415/ using document root <none> and password file <none>
10:08:55.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.347131 13703 fs_manager.cc:359] Metadata directory not provided
10:08:55.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.347247 13703 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:55.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.347601 13703 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:08:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.351011 13703 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data/instance:
10:08:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d108d647d787449e8ccfa107e6d0e438"
10:08:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:55 on dist-test-slave-ww78"
10:08:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.351964 13703 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal/instance:
10:08:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d108d647d787449e8ccfa107e6d0e438"
10:08:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:55 on dist-test-slave-ww78"
10:08:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.358266 13703 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.002s
10:08:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.361541 13718 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.362864 13703 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.362991 13703 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d108d647d787449e8ccfa107e6d0e438"
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:55 on dist-test-slave-ww78"
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.363152 13703 fs_report.cc:383] FS layout report
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data/data
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.386302 13703 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.387127 13703 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.387360 13703 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:55.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.390306 13703 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:08:55.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.390403 13703 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:08:55.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.390528 13703 ts_tablet_manager.cc:590] Registered 0 tablets
10:08:55.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.390743 13703 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
10:08:55.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.454879 13703 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.67:36725
10:08:55.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.454902 13839 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.67:36725 every 8 connection(s)
10:08:55.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.456558 13703 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/data/info.pb
10:08:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.459233 13841 heartbeater.cc:347] Connected to a master server at 127.12.205.126:37355
10:08:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.459615 13841 heartbeater.cc:464] Registering TS with master...
10:08:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.460878 13841 heartbeater.cc:511] Master 127.12.205.126:37355 requested a full tablet report, sending...
10:08:55.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.462160 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13703
10:08:55.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.462350 13109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/ts-2/wal/instance
10:08:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.466786 13840 heartbeater.cc:347] Connected to a master server at 127.12.205.124:46319
10:08:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.466933 13840 heartbeater.cc:464] Registering TS with master...
10:08:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.467502 13840 heartbeater.cc:511] Master 127.12.205.124:46319 requested a full tablet report, sending...
10:08:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.469295 13312 ts_manager.cc:194] Registered new tserver with Master: d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:08:55.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.473320 13151 ts_manager.cc:194] Registered new tserver with Master: d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:08:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.474193 13151 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.205.67:52047
10:08:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.477710 13842 heartbeater.cc:347] Connected to a master server at 127.12.205.125:45397
10:08:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.477864 13842 heartbeater.cc:464] Registering TS with master...
10:08:55.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.478260 13842 heartbeater.cc:511] Master 127.12.205.125:45397 requested a full tablet report, sending...
10:08:55.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.479281 13232 ts_manager.cc:194] Registered new tserver with Master: d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:08:55.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.489084 13109 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
10:08:55.491 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:08:55.492 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
10:08:55.493 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.205.126"
port: 37355
}
bound_http_address {
host: "127.12.205.126"
port: 41957
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.205.125"
port: 45397
}
bound_http_address {
host: "127.12.205.125"
port: 36539
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.205.124"
port: 46319
}
bound_http_address {
host: "127.12.205.124"
port: 42833
}
}
}
10:08:55.493 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:08:55.494 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.205.65"
port: 43323
}
bound_http_address {
host: "127.12.205.65"
port: 44393
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.205.66"
port: 37261
}
bound_http_address {
host: "127.12.205.66"
port: 44963
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.205.67"
port: 36725
}
bound_http_address {
host: "127.12.205.67"
port: 40415
}
}
}
10:08:55.494 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
10:08:55.496 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
10:08:55.498 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.205.126:37355(127.12.205.126:37355)] connecting to peer
10:08:55.499 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.205.124:46319(127.12.205.124:46319)] connecting to peer
10:08:55.500 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.205.125:45397(127.12.205.125:45397)] connecting to peer
10:08:55.502 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.205.124:46319(127.12.205.124:46319)] Successfully connected to peer
10:08:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.503556 13558 heartbeater.cc:503] Master 127.12.205.126:37355 was elected leader, sending a full tablet report...
10:08:55.505 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.205.124
10:08:55.514 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.205.126:37355(127.12.205.126:37355)] Successfully connected to peer
10:08:55.516 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.205.125:45397(127.12.205.125:45397)] Successfully connected to peer
10:08:55.517 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.205.126
10:08:55.521 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x96954393, L:/127.0.0.1:55304 - R:/127.12.205.124:46319] using SASL/PLAIN
10:08:55.522 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.205.125
10:08:55.539 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5afd8c25, L:/127.0.0.1:51818 - R:/127.12.205.126:37355] using SASL/PLAIN
10:08:55.540 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x49f1756f, L:/127.0.0.1:59114 - R:/127.12.205.125:45397] using SASL/PLAIN
10:08:55.541 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:08:55.542 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=44), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.126:37355, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.124:46319, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.125:45397, [26ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.124:46319: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.126:37355: OK, deferred=null) after lookup
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.543900 13151 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51818:
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.546291 13151 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:08:55.562 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 0863afe6c1a94168ae60e655f6873acd
10:08:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.613361 13488 tablet_service.cc:1467] Processing CreateTablet for tablet 81eb505f98c6437a81d854a52758ae11 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:08:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.613827 13769 tablet_service.cc:1467] Processing CreateTablet for tablet 81eb505f98c6437a81d854a52758ae11 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:08:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.614876 13488 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81eb505f98c6437a81d854a52758ae11. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.616130 13769 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81eb505f98c6437a81d854a52758ae11. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.615078 13768 tablet_service.cc:1467] Processing CreateTablet for tablet f3b49b0b82b044be83d4797336eecfea (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:08:55.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.617022 13768 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f3b49b0b82b044be83d4797336eecfea. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:55.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.616634 13632 tablet_service.cc:1467] Processing CreateTablet for tablet f3b49b0b82b044be83d4797336eecfea (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:08:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.617839 13632 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f3b49b0b82b044be83d4797336eecfea. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.617841 13487 tablet_service.cc:1467] Processing CreateTablet for tablet f3b49b0b82b044be83d4797336eecfea (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
10:08:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.618238 13487 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f3b49b0b82b044be83d4797336eecfea. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.616351 13633 tablet_service.cc:1467] Processing CreateTablet for tablet 81eb505f98c6437a81d854a52758ae11 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
10:08:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.625000 13633 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81eb505f98c6437a81d854a52758ae11. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.636287 13867 tablet_bootstrap.cc:492] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1: Bootstrap starting.
10:08:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.639139 13865 tablet_bootstrap.cc:492] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438: Bootstrap starting.
10:08:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.639838 13866 tablet_bootstrap.cc:492] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78: Bootstrap starting.
10:08:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.640975 13867 tablet_bootstrap.cc:654] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1: Neither blocks nor log segments found. Creating new log.
10:08:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.642215 13865 tablet_bootstrap.cc:654] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438: Neither blocks nor log segments found. Creating new log.
10:08:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.642246 13867 log.cc:824] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1: Log is configured to *not* fsync() on all Append() calls
10:08:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.643092 13866 tablet_bootstrap.cc:654] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78: Neither blocks nor log segments found. Creating new log.
10:08:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.643339 13865 log.cc:824] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438: Log is configured to *not* fsync() on all Append() calls
10:08:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.644223 13866 log.cc:824] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78: Log is configured to *not* fsync() on all Append() calls
10:08:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.645100 13867 tablet_bootstrap.cc:492] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1: No bootstrap required, opened a new log
10:08:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.645391 13867 ts_tablet_manager.cc:1367] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
10:08:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.645823 13865 tablet_bootstrap.cc:492] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438: No bootstrap required, opened a new log
10:08:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.646066 13865 ts_tablet_manager.cc:1367] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
10:08:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.646837 13866 tablet_bootstrap.cc:492] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78: No bootstrap required, opened a new log
10:08:55.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.647094 13866 ts_tablet_manager.cc:1367] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
10:08:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.651942 13865 raft_consensus.cc:348] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.652290 13865 raft_consensus.cc:374] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.652413 13865 raft_consensus.cc:729] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d108d647d787449e8ccfa107e6d0e438, State: Initialized, Role: FOLLOWER
10:08:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.653040 13865 consensus_queue.cc:260] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.654238 13867 raft_consensus.cc:348] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.654731 13867 raft_consensus.cc:374] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:55.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.655198 13867 raft_consensus.cc:729] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d61405dbc7b04c15a334468216058bc1, State: Initialized, Role: FOLLOWER
10:08:55.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.655424 13866 raft_consensus.cc:348] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.655826 13866 raft_consensus.cc:374] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.655913 13866 raft_consensus.cc:729] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5e999584f2f6417c9a44626007db7e78, State: Initialized, Role: FOLLOWER
10:08:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.656447 13867 consensus_queue.cc:260] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.656877 13865 ts_tablet_manager.cc:1398] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438: Time spent starting tablet: real 0.011s user 0.008s sys 0.000s
10:08:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.656893 13866 consensus_queue.cc:260] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.657477 13865 tablet_bootstrap.cc:492] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438: Bootstrap starting.
10:08:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.657847 13841 heartbeater.cc:503] Master 127.12.205.126:37355 was elected leader, sending a full tablet report...
10:08:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.660329 13865 tablet_bootstrap.cc:654] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438: Neither blocks nor log segments found. Creating new log.
10:08:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.668138 13867 ts_tablet_manager.cc:1398] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1: Time spent starting tablet: real 0.023s user 0.012s sys 0.008s
10:08:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.668725 13699 heartbeater.cc:503] Master 127.12.205.126:37355 was elected leader, sending a full tablet report...
10:08:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.668793 13865 tablet_bootstrap.cc:492] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438: No bootstrap required, opened a new log
10:08:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.668897 13867 tablet_bootstrap.cc:492] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1: Bootstrap starting.
10:08:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.668933 13865 ts_tablet_manager.cc:1367] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.005s
10:08:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.669708 13865 raft_consensus.cc:348] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.670044 13865 raft_consensus.cc:374] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:55.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.670222 13865 raft_consensus.cc:729] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d108d647d787449e8ccfa107e6d0e438, State: Initialized, Role: FOLLOWER
10:08:55.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.670542 13867 tablet_bootstrap.cc:654] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1: Neither blocks nor log segments found. Creating new log.
10:08:55.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.670547 13865 consensus_queue.cc:260] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.672088 13865 ts_tablet_manager.cc:1398] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
10:08:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.676113 13866 ts_tablet_manager.cc:1398] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78: Time spent starting tablet: real 0.029s user 0.015s sys 0.005s
10:08:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.676961 13866 tablet_bootstrap.cc:492] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78: Bootstrap starting.
10:08:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.679741 13866 tablet_bootstrap.cc:654] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78: Neither blocks nor log segments found. Creating new log.
10:08:55.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.681735 13867 tablet_bootstrap.cc:492] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1: No bootstrap required, opened a new log
10:08:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.681910 13867 ts_tablet_manager.cc:1367] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1: Time spent bootstrapping tablet: real 0.013s user 0.012s sys 0.000s
10:08:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.682487 13866 tablet_bootstrap.cc:492] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78: No bootstrap required, opened a new log
10:08:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.682626 13866 ts_tablet_manager.cc:1367] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
10:08:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.682667 13867 raft_consensus.cc:348] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.682932 13867 raft_consensus.cc:374] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.683024 13867 raft_consensus.cc:729] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d61405dbc7b04c15a334468216058bc1, State: Initialized, Role: FOLLOWER
10:08:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.683240 13867 consensus_queue.cc:260] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.683722 13867 ts_tablet_manager.cc:1398] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
10:08:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.683579 13866 raft_consensus.cc:348] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.684319 13866 raft_consensus.cc:374] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.684685 13866 raft_consensus.cc:729] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5e999584f2f6417c9a44626007db7e78, State: Initialized, Role: FOLLOWER
10:08:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.684979 13866 consensus_queue.cc:260] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.685467 13866 ts_tablet_manager.cc:1398] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
10:08:55.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.705381 13873 raft_consensus.cc:482] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:08:55.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.705652 13873 raft_consensus.cc:504] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.707396 13873 leader_election.cc:288] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725), d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:55.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.714388 13653 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81eb505f98c6437a81d854a52758ae11" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d61405dbc7b04c15a334468216058bc1" is_pre_election: true
10:08:55.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.714447 13789 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81eb505f98c6437a81d854a52758ae11" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d108d647d787449e8ccfa107e6d0e438" is_pre_election: true
10:08:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.714859 13789 raft_consensus.cc:2454] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 0.
10:08:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.714869 13653 raft_consensus.cc:2454] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 0.
10:08:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.715392 13435 leader_election.cc:302] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5e999584f2f6417c9a44626007db7e78, d108d647d787449e8ccfa107e6d0e438; no voters:
10:08:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.715729 13873 raft_consensus.cc:2781] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:08:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.715868 13873 raft_consensus.cc:482] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:08:55.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.715979 13873 raft_consensus.cc:3037] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Advancing to term 1
10:08:55.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.718044 13873 raft_consensus.cc:504] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.718667 13873 leader_election.cc:288] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 election: Requested vote from peers d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725), d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:55.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.719143 13789 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81eb505f98c6437a81d854a52758ae11" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d108d647d787449e8ccfa107e6d0e438"
10:08:55.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.719240 13653 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81eb505f98c6437a81d854a52758ae11" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d61405dbc7b04c15a334468216058bc1"
10:08:55.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.719362 13789 raft_consensus.cc:3037] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Advancing to term 1
10:08:55.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.719408 13653 raft_consensus.cc:3037] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Advancing to term 1
10:08:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.721793 13843 tablet.cc:2369] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:08:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.722435 13653 raft_consensus.cc:2454] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 1.
10:08:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.722443 13789 raft_consensus.cc:2454] T 81eb505f98c6437a81d854a52758ae11 P d108d647d787449e8ccfa107e6d0e438 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 1.
10:08:55.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.722853 13436 leader_election.cc:302] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5e999584f2f6417c9a44626007db7e78, d61405dbc7b04c15a334468216058bc1; no voters:
10:08:55.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.723140 13873 raft_consensus.cc:2781] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 1 FOLLOWER]: Leader election won for term 1
10:08:55.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.723526 13873 raft_consensus.cc:686] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [term 1 LEADER]: Becoming Leader. State: Replica: 5e999584f2f6417c9a44626007db7e78, State: Running, Role: LEADER
10:08:55.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.724012 13873 consensus_queue.cc:237] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } }
10:08:55.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.726641 13701 tablet.cc:2369] T 81eb505f98c6437a81d854a52758ae11 P d61405dbc7b04c15a334468216058bc1: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:08:55.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.729604 13151 catalog_manager.cc:5484] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5e999584f2f6417c9a44626007db7e78 (127.12.205.65). New cstate: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } health_report { overall_health: UNKNOWN } } }
10:08:55.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:55.747355 13560 tablet.cc:2369] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:08:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.996068 13871 raft_consensus.cc:482] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:08:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.996261 13871 raft_consensus.cc:504] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:55.997948 13871 leader_election.cc:288] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323), d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.004251 13653 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f3b49b0b82b044be83d4797336eecfea" candidate_uuid: "d108d647d787449e8ccfa107e6d0e438" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d61405dbc7b04c15a334468216058bc1" is_pre_election: true
10:08:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.004525 13653 raft_consensus.cc:2454] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d108d647d787449e8ccfa107e6d0e438 in term 0.
10:08:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.004523 13511 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f3b49b0b82b044be83d4797336eecfea" candidate_uuid: "d108d647d787449e8ccfa107e6d0e438" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e999584f2f6417c9a44626007db7e78" is_pre_election: true
10:08:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.004930 13511 raft_consensus.cc:2454] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d108d647d787449e8ccfa107e6d0e438 in term 0.
10:08:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.004987 13720 leader_election.cc:302] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d108d647d787449e8ccfa107e6d0e438, d61405dbc7b04c15a334468216058bc1; no voters:
10:08:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.005386 13871 raft_consensus.cc:2781] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:08:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.005493 13871 raft_consensus.cc:482] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:08:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.005558 13871 raft_consensus.cc:3037] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Advancing to term 1
10:08:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.007577 13871 raft_consensus.cc:504] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.008136 13871 leader_election.cc:288] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [CANDIDATE]: Term 1 election: Requested vote from peers 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323), d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.008601 13511 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f3b49b0b82b044be83d4797336eecfea" candidate_uuid: "d108d647d787449e8ccfa107e6d0e438" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e999584f2f6417c9a44626007db7e78"
10:08:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.008641 13653 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f3b49b0b82b044be83d4797336eecfea" candidate_uuid: "d108d647d787449e8ccfa107e6d0e438" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d61405dbc7b04c15a334468216058bc1"
10:08:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.008765 13511 raft_consensus.cc:3037] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Advancing to term 1
10:08:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.008837 13653 raft_consensus.cc:3037] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Advancing to term 1
10:08:56.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.010704 13653 raft_consensus.cc:2454] T f3b49b0b82b044be83d4797336eecfea P d61405dbc7b04c15a334468216058bc1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d108d647d787449e8ccfa107e6d0e438 in term 1.
10:08:56.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.010663 13511 raft_consensus.cc:2454] T f3b49b0b82b044be83d4797336eecfea P 5e999584f2f6417c9a44626007db7e78 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d108d647d787449e8ccfa107e6d0e438 in term 1.
10:08:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.011098 13720 leader_election.cc:302] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d108d647d787449e8ccfa107e6d0e438, d61405dbc7b04c15a334468216058bc1; no voters:
10:08:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.011382 13871 raft_consensus.cc:2781] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 1 FOLLOWER]: Leader election won for term 1
10:08:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.011773 13871 raft_consensus.cc:686] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [term 1 LEADER]: Becoming Leader. State: Replica: d108d647d787449e8ccfa107e6d0e438, State: Running, Role: LEADER
10:08:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.012303 13871 consensus_queue.cc:237] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.017005 13151 catalog_manager.cc:5484] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 reported cstate change: term changed from 0 to 1, leader changed from <none> to d108d647d787449e8ccfa107e6d0e438 (127.12.205.67). New cstate: current_term: 1 leader_uuid: "d108d647d787449e8ccfa107e6d0e438" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } health_report { overall_health: HEALTHY } } }
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.149063 13151 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51826:
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:08:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:08:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.149991 13151 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:08:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.161124 13632 tablet_service.cc:1467] Processing CreateTablet for tablet cd7c25479967477897245fc00dae88ee (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:08:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.162110 13632 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd7c25479967477897245fc00dae88ee. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.161667 13487 tablet_service.cc:1467] Processing CreateTablet for tablet cd7c25479967477897245fc00dae88ee (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:08:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.162385 13487 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd7c25479967477897245fc00dae88ee. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.167321 13867 tablet_bootstrap.cc:492] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: Bootstrap starting.
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.167687 13768 tablet_service.cc:1467] Processing CreateTablet for tablet cd7c25479967477897245fc00dae88ee (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.168569 13768 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd7c25479967477897245fc00dae88ee. 1 dirs total, 0 dirs full, 0 dirs failed
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.168857 13867 tablet_bootstrap.cc:654] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: Neither blocks nor log segments found. Creating new log.
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.168787 13873 consensus_queue.cc:1035] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.168946 13866 tablet_bootstrap.cc:492] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: Bootstrap starting.
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.170418 13866 tablet_bootstrap.cc:654] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: Neither blocks nor log segments found. Creating new log.
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.170727 13867 tablet_bootstrap.cc:492] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: No bootstrap required, opened a new log
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.170842 13867 ts_tablet_manager.cc:1367] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.171392 13867 raft_consensus.cc:348] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.171593 13867 raft_consensus.cc:374] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.171665 13867 raft_consensus.cc:729] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d61405dbc7b04c15a334468216058bc1, State: Initialized, Role: FOLLOWER
10:08:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.171976 13867 consensus_queue.cc:260] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.172549 13866 tablet_bootstrap.cc:492] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: No bootstrap required, opened a new log
10:08:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.172427 13872 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.172706 13866 ts_tablet_manager.cc:1367] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
10:08:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.172730 13872 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.173359 13866 raft_consensus.cc:348] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.173588 13866 raft_consensus.cc:374] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.173658 13866 raft_consensus.cc:729] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5e999584f2f6417c9a44626007db7e78, State: Initialized, Role: FOLLOWER
10:08:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.174144 13866 consensus_queue.cc:260] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.174546 13873 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.174752 13873 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.178328 13867 ts_tablet_manager.cc:1398] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
10:08:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.178488 13866 ts_tablet_manager.cc:1398] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
10:08:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.186733 13873 consensus_queue.cc:1035] T 81eb505f98c6437a81d854a52758ae11 P 5e999584f2f6417c9a44626007db7e78 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:56.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.198540 13900 tablet_bootstrap.cc:492] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: Bootstrap starting.
10:08:56.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.200362 13900 tablet_bootstrap.cc:654] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: Neither blocks nor log segments found. Creating new log.
10:08:56.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.202198 13900 tablet_bootstrap.cc:492] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: No bootstrap required, opened a new log
10:08:56.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.202454 13900 ts_tablet_manager.cc:1367] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
10:08:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.203109 13900 raft_consensus.cc:348] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.203294 13900 raft_consensus.cc:374] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:08:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.203380 13900 raft_consensus.cc:729] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d108d647d787449e8ccfa107e6d0e438, State: Initialized, Role: FOLLOWER
10:08:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.203613 13900 consensus_queue.cc:260] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.204083 13871 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.204368 13871 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.205757 13900 ts_tablet_manager.cc:1398] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
10:08:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.294275 13873 raft_consensus.cc:482] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:08:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.294503 13873 raft_consensus.cc:504] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.295226 13873 leader_election.cc:288] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261), d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:08:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.295713 13653 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd7c25479967477897245fc00dae88ee" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d61405dbc7b04c15a334468216058bc1" is_pre_election: true
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.295753 13789 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd7c25479967477897245fc00dae88ee" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d108d647d787449e8ccfa107e6d0e438" is_pre_election: true
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.295912 13653 raft_consensus.cc:2454] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 0.
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.295991 13789 raft_consensus.cc:2454] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 0.
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.296296 13436 leader_election.cc:302] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5e999584f2f6417c9a44626007db7e78, d61405dbc7b04c15a334468216058bc1; no voters:
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.296609 13873 raft_consensus.cc:2781] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.296690 13873 raft_consensus.cc:482] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:08:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.296769 13873 raft_consensus.cc:3037] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 0 FOLLOWER]: Advancing to term 1
10:08:56.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.298498 13873 raft_consensus.cc:504] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.299036 13873 leader_election.cc:288] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 election: Requested vote from peers d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261), d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:08:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.299377 13653 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd7c25479967477897245fc00dae88ee" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d61405dbc7b04c15a334468216058bc1"
10:08:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.299440 13789 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd7c25479967477897245fc00dae88ee" candidate_uuid: "5e999584f2f6417c9a44626007db7e78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d108d647d787449e8ccfa107e6d0e438"
10:08:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.299573 13653 raft_consensus.cc:3037] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 0 FOLLOWER]: Advancing to term 1
10:08:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.299619 13789 raft_consensus.cc:3037] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 0 FOLLOWER]: Advancing to term 1
10:08:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.302311 13789 raft_consensus.cc:2454] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 1.
10:08:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.302325 13653 raft_consensus.cc:2454] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5e999584f2f6417c9a44626007db7e78 in term 1.
10:08:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.302675 13436 leader_election.cc:302] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5e999584f2f6417c9a44626007db7e78, d61405dbc7b04c15a334468216058bc1; no voters:
10:08:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.302901 13873 raft_consensus.cc:2781] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 1 FOLLOWER]: Leader election won for term 1
10:08:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.303051 13873 raft_consensus.cc:686] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [term 1 LEADER]: Becoming Leader. State: Replica: 5e999584f2f6417c9a44626007db7e78, State: Running, Role: LEADER
10:08:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.303321 13873 consensus_queue.cc:237] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } }
10:08:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.304908 13879 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: TxnStatusTablet state changed. Reason: New leader 5e999584f2f6417c9a44626007db7e78. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.305106 13879 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: This TxnStatusTablet replica's current role is: LEADER
10:08:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.305697 13907 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
10:08:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.305968 13907 txn_status_manager.cc:929] Loading transaction status metadata into memory...
10:08:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.307102 13149 catalog_manager.cc:5484] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5e999584f2f6417c9a44626007db7e78 (127.12.205.65). New cstate: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } health_report { overall_health: UNKNOWN } } }
10:08:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.339949 13789 raft_consensus.cc:1261] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438 [term 1 FOLLOWER]: Refusing update from remote peer 5e999584f2f6417c9a44626007db7e78: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:08:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.339954 13653 raft_consensus.cc:1261] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1 [term 1 FOLLOWER]: Refusing update from remote peer 5e999584f2f6417c9a44626007db7e78: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:08:56.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.340649 13879 consensus_queue.cc:1035] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:56.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.340982 13873 consensus_queue.cc:1035] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:56.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.352603 13871 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: TxnStatusTablet state changed. Reason: New leader 5e999584f2f6417c9a44626007db7e78. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.352639 13872 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: TxnStatusTablet state changed. Reason: New leader 5e999584f2f6417c9a44626007db7e78. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.352891 13872 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.352896 13871 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.356045 13901 mvcc.cc:205] Tried to move back new op lower bound from 7021060654442225664 to 7021060654300991488. Current Snapshot: MvccSnapshot[applied={T|T < 7021060654442225664}]
10:08:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.359164 13879 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.359391 13879 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: This TxnStatusTablet replica's current role is: LEADER
10:08:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.359558 13871 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.359758 13871 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P d108d647d787449e8ccfa107e6d0e438: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.365005 13873 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.365270 13873 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P 5e999584f2f6417c9a44626007db7e78: This TxnStatusTablet replica's current role is: LEADER
10:08:56.366 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
10:08:56.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.369939 13905 mvcc.cc:205] Tried to move back new op lower bound from 7021060654442225664 to 7021060654300991488. Current Snapshot: MvccSnapshot[applied={T|T < 7021060654442225664}]
10:08:56.370 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 0863afe6c1a94168ae60e655f6873acd
10:08:56.370 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.205.126:37355
10:08:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.370674 13872 tablet_replica.cc:439] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5e999584f2f6417c9a44626007db7e78" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 } } peers { permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 } } peers { permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" member_type: VOTER last_known_addr { host: "127.12.205.67" port: 36725 } } }
10:08:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.370903 13872 tablet_replica.cc:442] T cd7c25479967477897245fc00dae88ee P d61405dbc7b04c15a334468216058bc1: This TxnStatusTablet replica's current role is: FOLLOWER
10:08:56.371 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:08:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.371873 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13120
10:08:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.374362 13191 generic_service.cc:196] Checking for leaks (request via RPC)
10:08:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.418984 13871 consensus_queue.cc:1035] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d61405dbc7b04c15a334468216058bc1" member_type: VOTER last_known_addr { host: "127.12.205.66" port: 37261 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.429116 13871 consensus_queue.cc:1035] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e999584f2f6417c9a44626007db7e78" member_type: VOTER last_known_addr { host: "127.12.205.65" port: 43323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:08:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.535461 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13120
10:08:56.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.553838 13841 heartbeater.cc:644] Failed to heartbeat to 127.12.205.126:37355 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.12.205.126:37355 (error 108)
10:08:56.554 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.205.126:37355(127.12.205.126:37355)] handling channelInactive
10:08:56.554 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.126:37355(127.12.205.126:37355)] cleaning up while in state READY due to: connection closed
10:08:56.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.554152 13699 heartbeater.cc:644] Failed to heartbeat to 127.12.205.126:37355 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.12.205.126:37355 (error 108)
10:08:56.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.554749 13558 heartbeater.cc:644] Failed to heartbeat to 127.12.205.126:37355 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.12.205.126:37355 (error 108)
10:08:56.564 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:08:56.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.613687 13920 raft_consensus.cc:482] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a63e2d0f1f62473395e18c56e6b648bd)
10:08:56.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.614012 13920 raft_consensus.cc:504] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:56.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.614379 13921 raft_consensus.cc:482] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a63e2d0f1f62473395e18c56e6b648bd)
10:08:56.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.614602 13921 raft_consensus.cc:504] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:56.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.616035 13921 leader_election.cc:288] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397)
10:08:56.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.616389 13322 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42436150052e4f878ad40cb5c962c74f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "051c8147615a410d851ef3d6db946e03" is_pre_election: true
10:08:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.616653 13322 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42436150052e4f878ad40cb5c962c74f in term 1.
10:08:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.617115 13920 leader_election.cc:288] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:08:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.617173 13218 leader_election.cc:302] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 051c8147615a410d851ef3d6db946e03, 42436150052e4f878ad40cb5c962c74f; no voters:
10:08:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.617553 13920 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 1 FOLLOWER]: Leader pre-election won for term 2
10:08:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.617763 13920 raft_consensus.cc:482] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 1 FOLLOWER]: Starting leader election (detected failure of leader a63e2d0f1f62473395e18c56e6b648bd)
10:08:56.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.617931 13920 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 1 FOLLOWER]: Advancing to term 2
10:08:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.621430 13920 raft_consensus.cc:504] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.622838 13322 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42436150052e4f878ad40cb5c962c74f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "051c8147615a410d851ef3d6db946e03"
10:08:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.623037 13322 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 1 FOLLOWER]: Advancing to term 2
10:08:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.626129 13322 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42436150052e4f878ad40cb5c962c74f in term 2.
10:08:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.626835 13218 leader_election.cc:302] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 051c8147615a410d851ef3d6db946e03, 42436150052e4f878ad40cb5c962c74f; no voters:
10:08:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.627864 13241 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "051c8147615a410d851ef3d6db946e03" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "42436150052e4f878ad40cb5c962c74f" is_pre_election: true
10:08:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.628178 13241 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 051c8147615a410d851ef3d6db946e03 in current term 2: Already voted for candidate 42436150052e4f878ad40cb5c962c74f in this term.
10:08:56.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.628880 13297 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111)
10:08:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.630789 13216 leader_election.cc:334] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111)
10:08:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.631449 13920 leader_election.cc:288] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 2 election: Requested vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:08:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.631763 13920 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 FOLLOWER]: Leader election won for term 2
10:08:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.631991 13920 raft_consensus.cc:686] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 LEADER]: Becoming Leader. State: Replica: 42436150052e4f878ad40cb5c962c74f, State: Running, Role: LEADER
10:08:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.632191 13216 leader_election.cc:334] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111)
10:08:56.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.632931 13920 consensus_queue.cc:237] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.634757 13920 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42436150052e4f878ad40cb5c962c74f. Latest consensus state: current_term: 2 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.635141 13920 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: LEADER
10:08:56.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.636864 13927 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
10:08:56.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.637099 13297 leader_election.cc:334] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111)
10:08:56.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.637252 13297 leader_election.cc:302] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 051c8147615a410d851ef3d6db946e03; no voters: 42436150052e4f878ad40cb5c962c74f, a63e2d0f1f62473395e18c56e6b648bd
10:08:56.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.637601 13921 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
10:08:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.639403 13927 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]
10:08:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.639806 13927 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b]
10:08:56.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.641391 13927 tablet_loader.cc:96] loaded metadata for tablet 81eb505f98c6437a81d854a52758ae11 (table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd])
10:08:56.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.641747 13927 tablet_loader.cc:96] loaded metadata for tablet cd7c25479967477897245fc00dae88ee (table kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b])
10:08:56.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.642041 13927 tablet_loader.cc:96] loaded metadata for tablet f3b49b0b82b044be83d4797336eecfea (table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd])
10:08:56.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.642252 13927 catalog_manager.cc:1484] Initializing Kudu cluster ID...
10:08:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.643008 13927 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:08:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.643088 13927 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
10:08:56.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.644182 13927 catalog_manager.cc:1504] Loading token signing keys...
10:08:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.645028 13927 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Loaded TSK: 0
10:08:56.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.646032 13927 catalog_manager.cc:1514] Initializing in-progress tserver states...
10:08:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.734709 13216 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Couldn't send request to peer a63e2d0f1f62473395e18c56e6b648bd. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:08:56.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.750012 13322 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Refusing update from remote peer 42436150052e4f878ad40cb5c962c74f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
10:08:56.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.750656 13920 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [LEADER]: Connected to new peer: Peer: permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
10:08:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.752028 13921 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42436150052e4f878ad40cb5c962c74f. Latest consensus state: current_term: 2 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.752272 13921 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: FOLLOWER
10:08:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.753439 13925 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.753775 13925 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: LEADER
10:08:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.753880 13921 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.754063 13921 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: FOLLOWER
10:08:56.878 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.126:37355(127.12.205.126:37355)] connecting to peer
10:08:56.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355
10:08:56.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.205.126:37355(127.12.205.126:37355)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355
10:08:56.879 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.12.205.126:37355(127.12.205.126:37355) for tablet Kudu Master: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355
10:08:56.898 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.126:37355(127.12.205.126:37355)] connecting to peer
10:08:56.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355
10:08:56.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.126:37355(127.12.205.126:37355)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355
10:08:56.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.205.126:37355: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355
10:08:56.901 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:08:56.901 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] sending RPC to server master-127.12.205.126:37355, [1ms] received response from server master-127.12.205.126:37355: Network error: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.126:37355, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.124:46319, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.125:45397, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.126:37355: Network error: Failed to connect to peer master-127.12.205.126:37355(127.12.205.126:37355): Connection refused: /127.12.205.126:37355, [22ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.124:46319: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.125:45397: OK, deferred=Deferred@830826201(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@1eb656d2, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@37c43d47)) after lookup
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:56.933494 13231 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59132:
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:08:56.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:56.939836 13231 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:08:57.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:57.247444 13216 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Couldn't send request to peer a63e2d0f1f62473395e18c56e6b648bd. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:08:57.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:57.794126 13216 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Couldn't send request to peer a63e2d0f1f62473395e18c56e6b648bd. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:08:57.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:57.895296 13842 heartbeater.cc:503] Master 127.12.205.125:45397 was elected leader, sending a full tablet report...
10:08:58.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.269696 13216 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Couldn't send request to peer a63e2d0f1f62473395e18c56e6b648bd. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:08:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.381808 13216 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111) [suppressed 22 similar messages]
10:08:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.431694 13700 heartbeater.cc:503] Master 127.12.205.125:45397 was elected leader, sending a full tablet report...
10:08:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.438097 13559 heartbeater.cc:503] Master 127.12.205.125:45397 was elected leader, sending a full tablet report...
10:08:58.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 81eb505f98c6437a81d854a52758ae11 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
10:08:58.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet f3b49b0b82b044be83d4797336eecfea for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
10:08:58.570 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=81eb505f98c6437a81d854a52758ae11@[5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323)[L],d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725),d61405dbc7b04c15a334468216058bc1(127.12.205.66:37261)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=f3b49b0b82b044be83d4797336eecfea@[5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323),d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725)[L],d61405dbc7b04c15a334468216058bc1(127.12.205.66:37261)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
10:08:58.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.205.125:45397, [1ms] Sub RPC GetTableLocations: received response from server master-127.12.205.125:45397: OK, deferred=null) after lookup
10:08:58.571 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323)] connecting to peer
10:08:58.571 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323)] Successfully connected to peer
10:08:58.587 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc3e3f29c, L:/127.0.0.1:38914 - R:/127.12.205.65:43323] using TOKEN/null
10:08:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.591816 13474 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.12.205.126:37355)
10:08:58.622 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.205.126:37355
10:08:58.622 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.623178 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/logs
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/info.pb
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.126:37355
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.126
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41957
10:08:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:08:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:08:58.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.781963 13216 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355): Couldn't send request to peer a63e2d0f1f62473395e18c56e6b648bd. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:08:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.818888 13279 master_runner.cc:186] Error getting master registration for 127.12.205.126:37355: Network error: Client connection negotiation failed: client connection to 127.12.205.126:37355: connect: Connection refused (error 111),
10:08:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.819134 13279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:08:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.937908 13964 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:08:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.938252 13964 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:08:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.938355 13964 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945504 13964 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945596 13964 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945632 13964 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945680 13964 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945719 13964 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945777 13964 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:08:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.945816 13964 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34589/jwks.json
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.955420 13964 master_runner.cc:386] Master server non-default flags:
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:08:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.126:37355
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/info.pb
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.126
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41957
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/logs
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:08:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.956729 13964 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:08:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.958344 13964 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:08:58.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.968562 13971 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:58.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.968938 13972 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:58.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:58.969394 13974 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:08:58.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.969987 13964 server_base.cc:1034] running on GCE node
10:08:58.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.970624 13964 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:08:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.971942 13964 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:08:58.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.973132 13964 hybrid_clock.cc:648] HybridClock initialized: now 1714126138973077 us; error 46 us; skew 500 ppm
10:08:58.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.973487 13964 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:08:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.981386 13964 webserver.cc:460] Webserver started at http://127.12.205.126:41957/ using document root <none> and password file <none>
10:08:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.982066 13964 fs_manager.cc:359] Metadata directory not provided
10:08:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.982180 13964 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:08:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.987378 13964 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.004s sys 0.001s
10:08:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.989991 13980 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:08:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.991163 13964 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.991312 13964 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a63e2d0f1f62473395e18c56e6b648bd"
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:52 on dist-test-slave-ww78"
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:58.992092 13964 fs_report.cc:383] FS layout report
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/wal
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/data
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:08:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:08:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:08:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:08:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:08:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:08:59.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.017838 13964 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:08:59.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.018729 13964 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:08:59.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.019038 13964 kserver.cc:163] Server-wide thread pool size limit: 3276
10:08:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.037392 13964 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.126:37355
10:08:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.037431 14041 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.126:37355 every 8 connection(s)
10:08:59.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.038733 13964 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-0/data/info.pb
10:08:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.039542 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 13964
10:08:59.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:08:59.040 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.205.125:45397
10:08:59.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:08:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.040555 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13198
10:08:59.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.041576 13272 generic_service.cc:196] Checking for leaks (request via RPC)
10:08:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.046277 14042 sys_catalog.cc:263] Verifying existing consensus state
10:08:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.050707 14042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Bootstrap starting.
10:08:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.067725 14042 log.cc:824] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Log is configured to *not* fsync() on all Append() calls
10:08:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.079973 14042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:08:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.080366 14042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: Bootstrap complete.
10:08:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.087854 14042 raft_consensus.cc:348] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.088675 14042 raft_consensus.cc:729] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a63e2d0f1f62473395e18c56e6b648bd, State: Initialized, Role: FOLLOWER
10:08:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.089361 14042 consensus_queue.cc:260] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.090252 14046 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.090579 14046 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:08:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.091343 14042 sys_catalog.cc:564] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: configured and running, proceeding with master startup.
10:08:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.101588 14057 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:08:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.101670 14057 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: loading cluster ID for follower catalog manager: success
10:08:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.108873 14057 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: acquiring CA information for follower catalog manager: success
10:08:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.114887 14057 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:08:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.180429 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13198
10:08:59.207 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.205.125:45397(127.12.205.125:45397)] handling channelInactive
10:08:59.207 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.125:45397(127.12.205.125:45397)] cleaning up while in state READY due to: connection closed
10:08:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.208792 14060 negotiation.cc:337] Failed RPC negotiation. Trace:
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.128740 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.12.205.125:45397 (local address 127.0.0.1:58344)
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.128890 (+ 150us) negotiation.cc:107] Waiting for socket to connect
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.128907 (+ 17us) client_negotiation.cc:174] Beginning negotiation
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.128999 (+ 92us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.178429 (+ 49430us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.178434 (+ 5us) client_negotiation.cc:394] Received NEGOTIATE response from server
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.178466 (+ 32us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.178930 (+ 464us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.178946 (+ 16us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.208242 (+ 29296us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: BlockingRecv error: recv got EOF from 127.12.205.125:45397 (error 108)
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":52}
10:08:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.208982 13300 connection.cc:535] server connection from 127.0.0.1:55298 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:08:59.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.212463 14059 negotiation.cc:337] Failed RPC negotiation. Trace:
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.178300 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37648 (local address 127.12.205.126:37355)
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.180978 (+ 2678us) server_negotiation.cc:206] Beginning negotiation
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.180984 (+ 6us) server_negotiation.cc:399] Waiting for connection header
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.181004 (+ 20us) server_negotiation.cc:407] Connection header received
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.181062 (+ 58us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.181067 (+ 5us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.181144 (+ 77us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.181238 (+ 94us) server_negotiation.cc:226] Negotiated authn=CERTIFICATE
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:08:59.207183 (+ 25945us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37648: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:08:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2137}
10:08:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.210006 13964 master_runner.cc:186] Error getting master registration for 127.12.205.125:45397: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: BlockingRecv error: recv got EOF from 127.12.205.125:45397 (error 108),
10:08:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.216845 13964 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:08:59.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.453102 14046 raft_consensus.cc:482] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:08:59.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.453363 14046 raft_consensus.cc:504] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:59.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.455345 14046 leader_election.cc:288] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:08:59.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.456149 13981 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:59.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.464875 13322 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a63e2d0f1f62473395e18c56e6b648bd" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "051c8147615a410d851ef3d6db946e03" is_pre_election: true
10:08:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.466049 13981 leader_election.cc:334] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.466372 13983 leader_election.cc:302] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a63e2d0f1f62473395e18c56e6b648bd; no voters: 051c8147615a410d851ef3d6db946e03, 42436150052e4f878ad40cb5c962c74f
10:08:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.466822 14046 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
10:08:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.481253 14067 raft_consensus.cc:482] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 42436150052e4f878ad40cb5c962c74f)
10:08:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.481408 14067 raft_consensus.cc:504] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.481956 14067 leader_election.cc:288] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397)
10:08:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.484467 13297 leader_election.cc:334] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:59.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.488219 14007 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "051c8147615a410d851ef3d6db946e03" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "a63e2d0f1f62473395e18c56e6b648bd" is_pre_election: true
10:08:59.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.488660 14007 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 051c8147615a410d851ef3d6db946e03 in term 1.
10:08:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.489099 13297 leader_election.cc:302] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 051c8147615a410d851ef3d6db946e03, a63e2d0f1f62473395e18c56e6b648bd; no voters: 42436150052e4f878ad40cb5c962c74f
10:08:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.489357 14067 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Leader pre-election won for term 3
10:08:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.489468 14067 raft_consensus.cc:482] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 42436150052e4f878ad40cb5c962c74f)
10:08:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.489532 14067 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 2 FOLLOWER]: Advancing to term 3
10:08:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.491410 14067 raft_consensus.cc:504] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.491987 14067 leader_election.cc:288] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 3 election: Requested vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397)
10:08:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.492424 14007 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "051c8147615a410d851ef3d6db946e03" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "a63e2d0f1f62473395e18c56e6b648bd"
10:08:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.492600 14007 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 1 FOLLOWER]: Advancing to term 3
10:08:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.493248 13297 leader_election.cc:334] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111)
10:08:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.496573 14007 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 051c8147615a410d851ef3d6db946e03 in term 3.
10:08:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.496959 13297 leader_election.cc:302] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 051c8147615a410d851ef3d6db946e03, a63e2d0f1f62473395e18c56e6b648bd; no voters: 42436150052e4f878ad40cb5c962c74f
10:08:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.497212 14067 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 3 FOLLOWER]: Leader election won for term 3
10:08:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.497624 14067 raft_consensus.cc:686] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [term 3 LEADER]: Becoming Leader. State: Replica: 051c8147615a410d851ef3d6db946e03, State: Running, Role: LEADER
10:08:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.498070 14067 consensus_queue.cc:237] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:08:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.499580 14071 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 051c8147615a410d851ef3d6db946e03. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.499835 14071 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: LEADER
10:08:59.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.500466 14073 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
10:08:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.503213 14073 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]
10:08:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.503887 14073 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b]
10:08:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.505895 14073 tablet_loader.cc:96] loaded metadata for tablet 81eb505f98c6437a81d854a52758ae11 (table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd])
10:08:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.506321 14073 tablet_loader.cc:96] loaded metadata for tablet cd7c25479967477897245fc00dae88ee (table kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b])
10:08:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.506722 14073 tablet_loader.cc:96] loaded metadata for tablet f3b49b0b82b044be83d4797336eecfea (table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd])
10:08:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.506997 14073 catalog_manager.cc:1484] Initializing Kudu cluster ID...
10:08:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.507911 14073 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:08:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.507992 14073 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
10:08:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.509182 14073 catalog_manager.cc:1504] Loading token signing keys...
10:08:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.509945 14073 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03: Loaded TSK: 0
10:08:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.510658 14073 catalog_manager.cc:1514] Initializing in-progress tserver states...
10:08:59.538 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.125:45397(127.12.205.125:45397)] connecting to peer
10:08:59.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397
10:08:59.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.125:45397(127.12.205.125:45397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397
10:08:59.538 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.12.205.125:45397(127.12.205.125:45397) for tablet Kudu Master: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397
10:08:59.558 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.126:37355(127.12.205.126:37355)] connecting to peer
10:08:59.559 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.125:45397(127.12.205.125:45397)] connecting to peer
10:08:59.559 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.205.126:37355(127.12.205.126:37355)] Successfully connected to peer
10:08:59.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397
10:08:59.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.125:45397(127.12.205.125:45397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397
10:08:59.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.205.125:45397: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397
10:08:59.563 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:08:59.563 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=25), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [1ms] received response from server master-127.12.205.125:45397: Network error: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.126:37355, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.124:46319, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.125:45397, [22ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.125:45397: Network error: Failed to connect to peer master-127.12.205.125:45397(127.12.205.125:45397): Connection refused: /127.12.205.125:45397, [25ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.124:46319: OK, deferred=Deferred@75081587(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@66b426ed, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@66c9d0af)) after lookup
10:08:59.570 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x96370754, L:/127.0.0.1:37670 - R:/127.12.205.126:37355] using TOKEN/null
10:08:59.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.587178 14007 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Refusing update from remote peer 051c8147615a410d851ef3d6db946e03: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
10:08:59.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.588248 14071 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
10:08:59.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.592857 13699 heartbeater.cc:347] Connected to a master server at 127.12.205.126:37355
10:08:59.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.602694 13558 heartbeater.cc:347] Connected to a master server at 127.12.205.126:37355
10:08:59.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.604887 13297 consensus_peers.cc:485] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 -> Peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Couldn't send request to peer 42436150052e4f878ad40cb5c962c74f. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:08:59.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.606863 14087 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:59.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.607374 14087 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:08:59.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.608031 14046 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 051c8147615a410d851ef3d6db946e03. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:59.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.608281 14046 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:08:59.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.609354 13997 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d61405dbc7b04c15a334468216058bc1" instance_seqno: 1714126134965044) as {username='slave'} at 127.12.205.66:38087; Asking this server to re-register.
10:08:59.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.609922 14046 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:59.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.610095 14046 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:08:59.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.610225 13699 heartbeater.cc:464] Registering TS with master...
10:08:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.610909 14071 sys_catalog.cc:455] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:08:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.611155 14071 sys_catalog.cc:458] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 [sys.catalog]: This master's current role is: LEADER
10:08:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.611137 13996 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5e999584f2f6417c9a44626007db7e78" instance_seqno: 1714126134479738) as {username='slave'} at 127.12.205.65:44501; Asking this server to re-register.
10:08:59.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.611954 13558 heartbeater.cc:464] Registering TS with master...
10:08:59.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.612571 13996 ts_manager.cc:194] Registered new tserver with Master: d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:08:59.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.613241 13995 ts_manager.cc:194] Registered new tserver with Master: 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323)
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.616259 13313 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59736:
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:08:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:08:59.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:08:59.618337 13313 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:08:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.902949 13841 heartbeater.cc:347] Connected to a master server at 127.12.205.126:37355
10:08:59.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.904057 13995 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" instance_seqno: 1714126135423129) as {username='slave'} at 127.12.205.67:46155; Asking this server to re-register.
10:08:59.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.904568 13841 heartbeater.cc:464] Registering TS with master...
10:08:59.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:08:59.905611 13995 ts_manager.cc:194] Registered new tserver with Master: d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:09:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:00.109865 13297 consensus_peers.cc:485] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 -> Peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Couldn't send request to peer 42436150052e4f878ad40cb5c962c74f. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:09:00.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:00.589998 13557 heartbeater.cc:503] Master 127.12.205.124:46319 was elected leader, sending a full tablet report...
10:09:00.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:00.611438 13297 consensus_peers.cc:485] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 -> Peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Couldn't send request to peer 42436150052e4f878ad40cb5c962c74f. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:09:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:00.899456 13840 heartbeater.cc:503] Master 127.12.205.124:46319 was elected leader, sending a full tablet report...
10:09:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.087870 13297 consensus_peers.cc:485] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 -> Peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Couldn't send request to peer 42436150052e4f878ad40cb5c962c74f. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:09:01.213 [DEBUG - Time-limited test] (Connection.java:748) [peer d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725)] connecting to peer
10:09:01.214 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725)] Successfully connected to peer
10:09:01.224 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x491fd902, L:/127.0.0.1:35674 - R:/127.12.205.67:36725] using TOKEN/null
10:09:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.233359 13766 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.12.205.126:37355)
10:09:01.262 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.205.125:45397
10:09:01.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.263095 13109 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/logs
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/info.pb
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.125:45397
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.125
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36539
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
10:09:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
10:09:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.437105 13698 heartbeater.cc:503] Master 127.12.205.124:46319 was elected leader, sending a full tablet report...
10:09:01.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.553560 13297 consensus_peers.cc:485] T 00000000000000000000000000000000 P 051c8147615a410d851ef3d6db946e03 -> Peer 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397): Couldn't send request to peer 42436150052e4f878ad40cb5c962c74f. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:09:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.574378 14121 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:09:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.574676 14121 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:09:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.574796 14121 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582033 14121 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582149 14121 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582202 14121 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582266 14121 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582320 14121 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582391 14121 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:09:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.582438 14121 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34589/jwks.json
10:09:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.591658 14121 master_runner.cc:386] Master server non-default flags:
10:09:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.205.84:44977
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.205.126:37355,127.12.205.125:45397,127.12.205.124:46319
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.205.125:45397
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34589/jwks.json
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/info.pb
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.205.125
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36539
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/logs
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:09:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 058c4bd848277b49fc34b19c81c39a215d7f637f
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 09:35:23 UTC on efd1337a2aa3
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 258
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
10:09:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.593276 14121 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:09:01.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.595081 14121 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:09:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.604828 14128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:09:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.605077 14129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:09:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.605413 14131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:09:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.605911 14121 server_base.cc:1034] running on GCE node
10:09:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.606626 14121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:09:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.607991 14121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:09:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.609174 14121 hybrid_clock.cc:648] HybridClock initialized: now 1714126141609130 us; error 34 us; skew 500 ppm
10:09:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.609503 14121 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:09:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.611541 14121 webserver.cc:460] Webserver started at http://127.12.205.125:36539/ using document root <none> and password file <none>
10:09:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.612214 14121 fs_manager.cc:359] Metadata directory not provided
10:09:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.612325 14121 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:09:01.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.617434 14121 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
10:09:01.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.620126 14137 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:09:01.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.621318 14121 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.621472 14121 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data,/tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42436150052e4f878ad40cb5c962c74f"
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 10:08:53 on dist-test-slave-ww78"
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.622236 14121 fs_report.cc:383] FS layout report
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/wal
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/data
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:09:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:09:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:09:01.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.643035 14121 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:09:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.644001 14121 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:09:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.643936 13297 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.125:45397: connect: Connection refused (error 111) [suppressed 23 similar messages]
10:09:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.644284 14121 kserver.cc:163] Server-wide thread pool size limit: 3276
10:09:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.663069 14121 rpc_server.cc:307] RPC server started. Bound to: 127.12.205.125:45397
10:09:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.663107 14198 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.205.125:45397 every 8 connection(s)
10:09:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.664470 14121 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskblP3Kh/test-tmp/mini-kudu-cluster1615870798992812740/master-1/data/info.pb
10:09:01.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.666900 13109 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu as pid 14121
10:09:01.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:09:01.667 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.205.124:46319
10:09:01.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:09:01.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.667719 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13279
10:09:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.668845 13353 generic_service.cc:196] Checking for leaks (request via RPC)
10:09:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.671589 14199 sys_catalog.cc:263] Verifying existing consensus state
10:09:01.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.674906 14199 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Bootstrap starting.
10:09:01.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.689731 14199 log.cc:824] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Log is configured to *not* fsync() on all Append() calls
10:09:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.702797 14199 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:09:01.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.703204 14199 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Bootstrap complete.
10:09:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.710808 14199 raft_consensus.cc:348] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:09:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.711567 14199 raft_consensus.cc:729] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42436150052e4f878ad40cb5c962c74f, State: Initialized, Role: FOLLOWER
10:09:01.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.712186 14199 consensus_queue.cc:260] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:09:01.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.712982 14203 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:01.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.713310 14203 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: FOLLOWER
10:09:01.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.714000 14199 sys_catalog.cc:564] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: configured and running, proceeding with master startup.
10:09:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.724489 14214 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:09:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.724721 14214 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: loading cluster ID for follower catalog manager: success
10:09:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.729089 14214 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: acquiring CA information for follower catalog manager: success
10:09:01.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.733455 14214 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:09:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.806993 14164 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 2 FOLLOWER]: Advancing to term 3
10:09:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.813020 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13279
10:09:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.830585 14222 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.830986 14222 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: FOLLOWER
10:09:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.831415 14203 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 051c8147615a410d851ef3d6db946e03. Latest consensus state: current_term: 3 leader_uuid: "051c8147615a410d851ef3d6db946e03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.831668 14203 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: FOLLOWER
10:09:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.836366 14140 connection.cc:535] server connection from 127.0.0.1:58850 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:09:01.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:01.837642 14216 negotiation.cc:337] Failed RPC negotiation. Trace:
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.756134 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.12.205.124:46319 (local address 127.0.0.1:59750)
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.756299 (+ 165us) negotiation.cc:107] Waiting for socket to connect
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.756316 (+ 17us) client_negotiation.cc:174] Beginning negotiation
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.756439 (+ 123us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.814165 (+ 57726us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.814170 (+ 5us) client_negotiation.cc:394] Received NEGOTIATE response from server
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.814202 (+ 32us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
10:09:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.814655 (+ 453us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
10:09:01.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.814670 (+ 15us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
10:09:01.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:01.836512 (+ 21842us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
10:09:01.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":64}
10:09:01.838 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.205.124:46319(127.12.205.124:46319)] handling channelInactive
10:09:01.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.205.124:46319(127.12.205.124:46319)] cleaning up while in state READY due to: connection closed
10:09:01.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.838042 14121 master_runner.cc:186] Error getting master registration for 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
10:09:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.842911 14121 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:09:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.908066 13842 heartbeater.cc:347] Connected to a master server at 127.12.205.125:45397
10:09:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.910967 14154 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d108d647d787449e8ccfa107e6d0e438" instance_seqno: 1714126135423129) as {username='slave'} at 127.12.205.67:44371; Asking this server to re-register.
10:09:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.911665 13842 heartbeater.cc:464] Registering TS with master...
10:09:01.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:01.913326 14154 ts_manager.cc:194] Registered new tserver with Master: d108d647d787449e8ccfa107e6d0e438 (127.12.205.67:36725)
10:09:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.098613 14225 raft_consensus.cc:482] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 051c8147615a410d851ef3d6db946e03)
10:09:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.098793 14225 raft_consensus.cc:504] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:09:02.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.099488 14225 leader_election.cc:288] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 42436150052e4f878ad40cb5c962c74f (127.12.205.125:45397), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:09:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.101686 13983 leader_election.cc:334] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:09:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.105301 14164 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a63e2d0f1f62473395e18c56e6b648bd" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "42436150052e4f878ad40cb5c962c74f" is_pre_election: true
10:09:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.106144 13981 leader_election.cc:302] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a63e2d0f1f62473395e18c56e6b648bd; no voters: 051c8147615a410d851ef3d6db946e03, 42436150052e4f878ad40cb5c962c74f
10:09:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.106384 14225 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
10:09:02.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.116011 14203 raft_consensus.cc:482] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 051c8147615a410d851ef3d6db946e03)
10:09:02.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.116246 14203 raft_consensus.cc:504] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:09:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.118031 14203 leader_election.cc:288] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:09:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.118506 14007 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42436150052e4f878ad40cb5c962c74f" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "a63e2d0f1f62473395e18c56e6b648bd" is_pre_election: true
10:09:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.118636 14140 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:09:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.118753 14007 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42436150052e4f878ad40cb5c962c74f in term 3.
10:09:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.119285 14138 leader_election.cc:302] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42436150052e4f878ad40cb5c962c74f, a63e2d0f1f62473395e18c56e6b648bd; no voters:
10:09:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.119655 14203 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 3 FOLLOWER]: Leader pre-election won for term 4
10:09:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.119771 14203 raft_consensus.cc:482] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 3 FOLLOWER]: Starting leader election (detected failure of leader 051c8147615a410d851ef3d6db946e03)
10:09:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.119884 14203 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 3 FOLLOWER]: Advancing to term 4
10:09:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.120241 14140 leader_election.cc:334] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:09:02.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.122038 14203 raft_consensus.cc:504] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:09:02.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.122584 14203 leader_election.cc:288] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 4 election: Requested vote from peers a63e2d0f1f62473395e18c56e6b648bd (127.12.205.126:37355), 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319)
10:09:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.122952 14007 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42436150052e4f878ad40cb5c962c74f" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "a63e2d0f1f62473395e18c56e6b648bd"
10:09:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.123138 14007 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 3 FOLLOWER]: Advancing to term 4
10:09:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.123641 14140 leader_election.cc:334] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111)
10:09:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.125164 14007 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42436150052e4f878ad40cb5c962c74f in term 4.
10:09:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.125515 14138 leader_election.cc:302] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42436150052e4f878ad40cb5c962c74f, a63e2d0f1f62473395e18c56e6b648bd; no voters: 051c8147615a410d851ef3d6db946e03
10:09:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.125792 14203 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 4 FOLLOWER]: Leader election won for term 4
10:09:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.125948 14203 raft_consensus.cc:686] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [term 4 LEADER]: Becoming Leader. State: Replica: 42436150052e4f878ad40cb5c962c74f, State: Running, Role: LEADER
10:09:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.126327 14203 consensus_queue.cc:237] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } }
10:09:02.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.127578 14222 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42436150052e4f878ad40cb5c962c74f. Latest consensus state: current_term: 4 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:02.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.127733 14222 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: LEADER
10:09:02.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.128232 14230 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
10:09:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.130511 14230 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd]
10:09:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.130931 14230 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b]
10:09:02.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.132392 14230 tablet_loader.cc:96] loaded metadata for tablet 81eb505f98c6437a81d854a52758ae11 (table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd])
10:09:02.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.132849 14230 tablet_loader.cc:96] loaded metadata for tablet cd7c25479967477897245fc00dae88ee (table kudu_system.kudu_transactions [id=60e906e903fa4061934cd6752b41794b])
10:09:02.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.133150 14230 tablet_loader.cc:96] loaded metadata for tablet f3b49b0b82b044be83d4797336eecfea (table txn_manager_fallback_rolling [id=0863afe6c1a94168ae60e655f6873acd])
10:09:02.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.133360 14230 catalog_manager.cc:1484] Initializing Kudu cluster ID...
10:09:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.134135 14230 catalog_manager.cc:1259] Loaded cluster ID: 4dba6610b67f456baf0142f21f5aae78
10:09:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.134229 14230 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
10:09:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.135516 14230 catalog_manager.cc:1504] Loading token signing keys...
10:09:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.136294 14230 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f: Loaded TSK: 0
10:09:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.137106 14230 catalog_manager.cc:1514] Initializing in-progress tserver states...
10:09:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.203379 14007 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [term 4 FOLLOWER]: Refusing update from remote peer 42436150052e4f878ad40cb5c962c74f: Log matching property violated. Preceding OpId in replica: term: 3 index: 13. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
10:09:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.204043 14222 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [LEADER]: Connected to new peer: Peer: permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
10:09:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.205626 14225 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42436150052e4f878ad40cb5c962c74f. Latest consensus state: current_term: 4 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:02.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.205890 14225 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:09:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.207176 14203 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.207412 14203 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f [sys.catalog]: This master's current role is: LEADER
10:09:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.208352 14225 sys_catalog.cc:455] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "42436150052e4f878ad40cb5c962c74f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a63e2d0f1f62473395e18c56e6b648bd" member_type: VOTER last_known_addr { host: "127.12.205.126" port: 37355 } } peers { permanent_uuid: "42436150052e4f878ad40cb5c962c74f" member_type: VOTER last_known_addr { host: "127.12.205.125" port: 45397 } } peers { permanent_uuid: "051c8147615a410d851ef3d6db946e03" member_type: VOTER last_known_addr { host: "127.12.205.124" port: 46319 } } }
10:09:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.208586 14225 sys_catalog.cc:458] T 00000000000000000000000000000000 P a63e2d0f1f62473395e18c56e6b648bd [sys.catalog]: This master's current role is: FOLLOWER
10:09:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.208781 14140 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Couldn't send request to peer 051c8147615a410d851ef3d6db946e03. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:09:02.318 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.124:46319(127.12.205.124:46319)] connecting to peer
10:09:02.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319
10:09:02.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.124:46319(127.12.205.124:46319)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319
10:09:02.319 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.12.205.124:46319(127.12.205.124:46319) for tablet Kudu Master: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319
10:09:02.338 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.124:46319(127.12.205.124:46319)] connecting to peer
10:09:02.338 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.12.205.125:45397(127.12.205.125:45397)] connecting to peer
10:09:02.339 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319
10:09:02.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.205.124:46319(127.12.205.124:46319)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319
10:09:02.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.205.124:46319: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319
10:09:02.339 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.205.125:45397(127.12.205.125:45397)] Successfully connected to peer
10:09:02.350 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0928ad52, L:/127.0.0.1:58858 - R:/127.12.205.125:45397] using TOKEN/null
10:09:02.352 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:09:02.352 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=34), Traces: [0ms] sending RPC to server master-127.12.205.124:46319, [1ms] received response from server master-127.12.205.124:46319: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.126:37355, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.124:46319, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.125:45397, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.124:46319: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.126:37355: OK, [34ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.125:45397: OK, deferred=Deferred@2049088796(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@20824811, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@21cf55f4)) after lookup
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.390074 14154 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:58868:
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:09:02.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.393131 14234 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
10:09:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.445608 13700 heartbeater.cc:347] Connected to a master server at 127.12.205.125:45397
10:09:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.446628 14154 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d61405dbc7b04c15a334468216058bc1" instance_seqno: 1714126134965044) as {username='slave'} at 127.12.205.66:53095; Asking this server to re-register.
10:09:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.447225 13700 heartbeater.cc:464] Registering TS with master...
10:09:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.447492 13700 heartbeater.cc:511] Master 127.12.205.125:45397 requested a full tablet report, sending...
10:09:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.448889 14154 ts_manager.cc:194] Registered new tserver with Master: d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261)
10:09:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.569626 14184 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:58858 (request call id 1) took 216 ms (client timeout 216 ms). Trace:
10:09:02.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.569787 14184 rpcz_store.cc:269] 0426 10:09:02.353317 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:09:02.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:02.353402 (+ 85us) service_pool.cc:225] Handling call
10:09:02.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:02.569557 (+216155us) inbound_call.cc:173] Queueing success response
10:09:02.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":410,"init.queue_time_us":340,"thread_start_us":1042,"threads_started":5}
10:09:02.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.12.205.124:46319, [1ms] received response from server master-127.12.205.124:46319: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.126:37355, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.124:46319, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.125:45397, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.124:46319: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.126:37355: OK, [34ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.125:45397: OK, [35ms] sending RPC to server master-127.12.205.125:45397, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2049088796(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@20824811 -> (continuation of Deferred@763965896 after retry RPC@1127056152), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@21cf55f4 -> (continuation of Deferred@763965896 after retry RPC@1127056152)))
10:09:02.571 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.12.205.124:46319, [1ms] received response from server master-127.12.205.124:46319: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.126:37355, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.124:46319, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.205.125:45397, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.124:46319: Network error: Failed to connect to peer master-127.12.205.124:46319(127.12.205.124:46319): Connection refused: /127.12.205.124:46319, [21ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.126:37355: OK, [34ms] Sub RPC ConnectToMaster: received response from server master-127.12.205.125:45397: OK, [35ms] sending RPC to server master-127.12.205.125:45397, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2049088796(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@20824811 -> (continuation of Deferred@763965896 after retry RPC@1127056152), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@21cf55f4 -> (continuation of Deferred@763965896 after retry RPC@1127056152)))
10:09:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.593312 13559 heartbeater.cc:347] Connected to a master server at 127.12.205.125:45397
10:09:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.594727 14154 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5e999584f2f6417c9a44626007db7e78" instance_seqno: 1714126134479738) as {username='slave'} at 127.12.205.65:58423; Asking this server to re-register.
10:09:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.595360 13559 heartbeater.cc:464] Registering TS with master...
10:09:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.595619 13559 heartbeater.cc:511] Master 127.12.205.125:45397 requested a full tablet report, sending...
10:09:02.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.596850 14154 ts_manager.cc:194] Registered new tserver with Master: 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323)
10:09:02.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.704275 14140 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Couldn't send request to peer 051c8147615a410d851ef3d6db946e03. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:09:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.828794 14184 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:58858 (request call id 2) took 250 ms (client timeout 250 ms). Trace:
10:09:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.828907 14184 rpcz_store.cc:269] 0426 10:09:02.578636 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:09:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:02.578705 (+ 69us) service_pool.cc:225] Handling call
10:09:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:02.828739 (+250034us) inbound_call.cc:173] Queueing success response
10:09:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:09:02.829 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1901955500(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@2a660286, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@4e056173))
10:09:02.830 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1901955500(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@2a660286, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@4e056173))
10:09:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:02.950871 13441 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.144s from last keepalive heartbeat (effective timeout is 1.000s)
10:09:02.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.959180 14247 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
10:09:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:02.965694 14250 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
10:09:03.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.089304 14184 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:58858 (request call id 3) took 250 ms (client timeout 250 ms). Trace:
10:09:03.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.089423 14184 rpcz_store.cc:269] 0426 10:09:02.839148 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:09:03.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:02.839227 (+ 79us) service_pool.cc:225] Handling call
10:09:03.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:03.089235 (+250008us) inbound_call.cc:173] Queueing success response
10:09:03.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:09:03.090 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1629425384(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@7c12eb8e, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@66fab4b8))
10:09:03.090 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1629425384(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@7c12eb8e, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@66fab4b8))
10:09:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.163273 14140 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Couldn't send request to peer 051c8147615a410d851ef3d6db946e03. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:09:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.348804 14184 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:58858 (request call id 4) took 250 ms (client timeout 250 ms). Trace:
10:09:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.348973 14184 rpcz_store.cc:269] 0426 10:09:03.098599 (+ 0us) service_pool.cc:168] Inserting onto call queue
10:09:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:03.098674 (+ 75us) service_pool.cc:225] Handling call
10:09:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 10:09:03.348722 (+250048us) inbound_call.cc:173] Queueing success response
10:09:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
10:09:03.349 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@51466221(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@1a607929, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@7145bf70))
10:09:03.349 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.12.205.125:45397, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@51466221(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@1a607929, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@7145bf70))
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:03.407145 14153 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:58876:
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
10:09:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.407809 14153 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:09:03.421 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 5e999584f2f6417c9a44626007db7e78 (127.12.205.65:43323): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1714126136 last_transition_timestamp: 1714126142
10:09:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:03.694200 14140 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Couldn't send request to peer 051c8147615a410d851ef3d6db946e03. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
10:09:03.841 [DEBUG - main] (Connection.java:575) [peer 5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323)] disconnecting while in state READY
10:09:03.841 [DEBUG - main] (Connection.java:575) [peer master-127.12.205.126:37355(127.12.205.126:37355)] disconnecting while in state READY
10:09:03.841 [DEBUG - main] (Connection.java:575) [peer d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725)] disconnecting while in state READY
10:09:03.841 [DEBUG - main] (Connection.java:575) [peer master-127.12.205.124:46319(127.12.205.124:46319)] disconnecting while in state TERMINATED
10:09:03.841 [DEBUG - main] (Connection.java:575) [peer master-127.12.205.125:45397(127.12.205.125:45397)] disconnecting while in state READY
10:09:03.842 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323)] handling channelInactive
10:09:03.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5e999584f2f6417c9a44626007db7e78(127.12.205.65:43323)] cleaning up while in state READY due to: connection closed
10:09:03.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:09:03.843 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.205.126:37355(127.12.205.126:37355)] handling channelInactive
10:09:03.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.205.126:37355(127.12.205.126:37355)] cleaning up while in state READY due to: connection closed
10:09:03.843 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725)] handling channelInactive
10:09:03.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d108d647d787449e8ccfa107e6d0e438(127.12.205.67:36725)] cleaning up while in state READY due to: connection closed
10:09:03.844 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.205.125:45397(127.12.205.125:45397)] handling channelInactive
10:09:03.844 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.205.125:45397(127.12.205.125:45397)] cleaning up while in state READY due to: connection closed
10:09:03.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:03.844421 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13402
10:09:03.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:03.851627 13552 generic_service.cc:196] Checking for leaks (request via RPC)
10:09:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:03.915957 13842 heartbeater.cc:503] Master 127.12.205.125:45397 was elected leader, sending a full tablet report...
10:09:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:03.973510 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13402
10:09:03.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:03.996842 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13562
10:09:04.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.003229 13693 generic_service.cc:196] Checking for leaks (request via RPC)
10:09:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.097126 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13562
10:09:04.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:04.110783 13720 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.12.205.66:37261 (error 108)
10:09:04.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.110970 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13703
10:09:04.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:04.112357 13720 consensus_peers.cc:485] T f3b49b0b82b044be83d4797336eecfea P d108d647d787449e8ccfa107e6d0e438 -> Peer d61405dbc7b04c15a334468216058bc1 (127.12.205.66:37261): Couldn't send request to peer d61405dbc7b04c15a334468216058bc1. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.66:37261: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:09:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 10:09:04.115242 14140 consensus_peers.cc:485] T 00000000000000000000000000000000 P 42436150052e4f878ad40cb5c962c74f -> Peer 051c8147615a410d851ef3d6db946e03 (127.12.205.124:46319): Couldn't send request to peer 051c8147615a410d851ef3d6db946e03. Status: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
10:09:04.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.118192 13833 generic_service.cc:196] Checking for leaks (request via RPC)
10:09:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.215154 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13703
10:09:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.222707 13964 master_runner.cc:186] Error getting master registration for 127.12.205.124:46319: Network error: Client connection negotiation failed: client connection to 127.12.205.124:46319: connect: Connection refused (error 111),
10:09:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.222885 13964 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:09:04.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.231194 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 13964
10:09:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.237301 14037 generic_service.cc:196] Checking for leaks (request via RPC)
10:09:04.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.343631 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 13964
10:09:04.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.359843 13109 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu pid 14121
10:09:04.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.365339 14194 generic_service.cc:196] Checking for leaks (request via RPC)
10:09:04.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 10:09:04.475674 13109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskblP3Kh/build/asan/bin/kudu with pid 14121
10:09:04.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T10:09:04Z chronyd exiting
10:09:04.618 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)