07:05:43.357 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:05:43.357 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskIiZZ1l/test-tmp
07:05:43.358 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
07:05:43.358 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskIiZZ1l/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
07:05:43.359 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:05:43.658 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:05:43.658 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:05:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:43.660887 31208 mini_oidc.cc:152] Starting JWKS server
07:05:43.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:43.663272 31208 webserver.cc:460] Webserver started at http://127.0.0.1:34995/ using document root <none> and password file <none>
07:05:43.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:43.663470 31208 mini_oidc.cc:189] Starting OIDC Discovery server
07:05:43.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:43.663969 31208 webserver.cc:460] Webserver started at http://127.0.0.1:36319/ using document root <none> and password file <none>
07:05:43.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:05:43Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:05:43.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:05:43Z Disabled control of system clock
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:43.690485 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/logs
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/info.pb
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.62:42411
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.62
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:05:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.985833 31219 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.986111 31219 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.986197 31219 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993219 31219 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993309 31219 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993343 31219 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993388 31219 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993424 31219 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993471 31219 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:43.993502 31219 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.001947 31219 master_runner.cc:386] Master server non-default flags:
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.62:42411
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/info.pb
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.62
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/logs
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:44.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.003021 31219 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:44.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.004604 31219 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.014312 31224 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.014748 31225 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.015030 31227 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.015424 31219 server_base.cc:1034] running on GCE node
07:05:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.016096 31219 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.017323 31219 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:44.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.018523 31219 hybrid_clock.cc:648] HybridClock initialized: now 1714115144018477 us; error 34 us; skew 500 ppm
07:05:44.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.018850 31219 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:44.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.020888 31219 webserver.cc:460] Webserver started at http://127.30.122.62:45449/ using document root <none> and password file <none>
07:05:44.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.021523 31219 fs_manager.cc:359] Metadata directory not provided
07:05:44.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.021620 31219 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:44.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.021911 31219 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:05:44.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.024260 31219 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/instance:
07:05:44.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb"
07:05:44.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.025017 31219 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal/instance:
07:05:44.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb"
07:05:44.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.029605 31219 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.000s sys 0.004s
07:05:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.032542 31233 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:44.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.033634 31219 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.033789 31219 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb"
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.033989 31219 fs_report.cc:383] FS layout report
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/data
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.054106 31219 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.054926 31219 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.055204 31219 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.073987 31295 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.62:42411 every 8 connection(s)
07:05:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.074012 31219 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.62:42411
07:05:44.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.075331 31219 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/info.pb
07:05:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.078881 31296 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.081404 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 31219
07:05:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.081652 31208 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal/instance
07:05:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.083709 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal
07:05:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data
07:05:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/logs
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data/info.pb
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.61:45247
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.61
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:05:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.084323 31296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.098773 31296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.099682 31234 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:44.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.101092 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.61:45247: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:44.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.150434 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } attempt: 1
07:05:44.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.153091 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.61:45247: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:44.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.221365 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } attempt: 2
07:05:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.223167 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.61:45247: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:44.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.314373 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } attempt: 3
07:05:44.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.316205 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.61:45247: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.388442 31300 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.388738 31300 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.388850 31300 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.395972 31300 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.396073 31300 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.396111 31300 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.396159 31300 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.396196 31300 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.396245 31300 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.396293 31300 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.404932 31300 master_runner.cc:386] Master server non-default flags:
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.61:45247
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data/info.pb
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.61
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/logs
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.406085 31300 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.407724 31300 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.417624 31306 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.417874 31307 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.418175 31309 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.418696 31300 server_base.cc:1034] running on GCE node
07:05:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.419327 31300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.420677 31300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.421885 31300 hybrid_clock.cc:648] HybridClock initialized: now 1714115144421837 us; error 34 us; skew 500 ppm
07:05:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.422214 31300 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.424326 31300 webserver.cc:460] Webserver started at http://127.30.122.61:32891/ using document root <none> and password file <none>
07:05:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.424988 31300 fs_manager.cc:359] Metadata directory not provided
07:05:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.425087 31300 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.425381 31300 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:05:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.427822 31300 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data/instance:
07:05:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fea5e5cdb708445faa58d8fde854b964"
07:05:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.428634 31300 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal/instance:
07:05:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fea5e5cdb708445faa58d8fde854b964"
07:05:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.433290 31300 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.002s sys 0.003s
07:05:44.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.436312 31315 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.437492 31300 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.437644 31300 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fea5e5cdb708445faa58d8fde854b964"
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.437804 31300 fs_report.cc:383] FS layout report
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data/data
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.450137 31300 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.450779 31300 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:44.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.451002 31300 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:44.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.459424 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } attempt: 4
07:05:44.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.461316 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.61:45247: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.474738 31300 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.61:45247
07:05:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.474773 31376 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.61:45247 every 8 connection(s)
07:05:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.475977 31300 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/data/info.pb
07:05:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.479346 31377 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.479873 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 31300
07:05:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.480059 31208 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-1/wal/instance
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.481003 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/logs
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/info.pb
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.60:37495
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.60
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:05:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:05:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.484844 31377 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.497052 31377 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.505533 31377 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.506456 31318 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.507797 31377 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.557035 31377 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 1
07:05:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.558769 31377 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.626966 31377 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 2
07:05:44.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.628829 31377 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.720078 31377 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 3
07:05:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.721889 31377 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.760571 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } attempt: 5
07:05:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.766433 31296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.768198 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.783599 31379 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:44.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.783874 31379 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:44.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.783962 31379 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791114 31379 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791200 31379 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791236 31379 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791283 31379 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791319 31379 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791371 31379 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.791414 31379 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.799971 31379 master_runner.cc:386] Master server non-default flags:
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.60:37495
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/info.pb
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.60
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/logs
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.801108 31379 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.802654 31379 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:44.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.812803 31387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.813179 31388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.813647 31390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:44.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.814040 31379 server_base.cc:1034] running on GCE node
07:05:44.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.814690 31379 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:44.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.815930 31379 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:44.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.817129 31379 hybrid_clock.cc:648] HybridClock initialized: now 1714115144817086 us; error 33 us; skew 500 ppm
07:05:44.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.817467 31379 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.819406 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 1
07:05:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.819489 31379 webserver.cc:460] Webserver started at http://127.30.122.60:38757/ using document root <none> and password file <none>
07:05:44.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.820259 31379 fs_manager.cc:359] Metadata directory not provided
07:05:44.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.820380 31379 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:44.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.820681 31379 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:05:44.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.821034 31296 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.823083 31379 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/instance:
07:05:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a08014ec6614098a83031094f54a102"
07:05:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.823763 31379 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal/instance:
07:05:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a08014ec6614098a83031094f54a102"
07:05:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.828243 31379 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
07:05:44.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.831255 31396 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.832443 31379 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.832576 31379 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a08014ec6614098a83031094f54a102"
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.832728 31379 fs_report.cc:383] FS layout report
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/data
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:44.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.848083 31379 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:44.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.848817 31379 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:44.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.849042 31379 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:44.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.865088 31377 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 4
07:05:44.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.867125 31377 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.868494 31379 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.60:37495
07:05:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.868515 31458 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.60:37495 every 8 connection(s)
07:05:44.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.869870 31379 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/info.pb
07:05:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.873276 31459 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.874570 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 31379
07:05:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.874742 31208 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal/instance
07:05:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.880030 31459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.889240 31296 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 2
07:05:44.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.892108 31459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.903337 31459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } has no permanent_uuid. Determining permanent_uuid...
07:05:44.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.905675 31296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Bootstrap starting.
07:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.909178 31296 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Neither blocks nor log segments found. Creating new log.
07:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.910405 31296 log.cc:824] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Log is configured to *not* fsync() on all Append() calls
07:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.912808 31459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Bootstrap starting.
07:05:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.913177 31296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: No bootstrap required, opened a new log
07:05:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.916213 31459 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Neither blocks nor log segments found. Creating new log.
07:05:44.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.917346 31459 log.cc:824] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Log is configured to *not* fsync() on all Append() calls
07:05:44.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.920544 31459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: No bootstrap required, opened a new log
07:05:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.920992 31296 raft_consensus.cc:348] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.921231 31296 raft_consensus.cc:374] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.921311 31296 raft_consensus.cc:729] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 75aa2aca8a1d47b2b09e9bdd5474f9cb, State: Initialized, Role: FOLLOWER
07:05:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.921783 31296 consensus_queue.cc:260] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.922535 31469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.922842 31469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: FOLLOWER
07:05:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.923507 31296 sys_catalog.cc:564] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: configured and running, proceeding with master startup.
07:05:44.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.927189 31459 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.927443 31459 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:44.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.927546 31459 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a08014ec6614098a83031094f54a102, State: Initialized, Role: FOLLOWER
07:05:44.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.928118 31459 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.928985 31481 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.929400 31481 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:44.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.930022 31459 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: configured and running, proceeding with master startup.
07:05:44.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.934551 31480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:44.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:44.934661 31480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:44.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.938423 31469 raft_consensus.cc:482] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:44.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.938632 31469 raft_consensus.cc:504] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.940310 31469 leader_election.cc:288] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.931970 31219 master_runner.cc:186] Error getting master registration for 127.30.122.61:45247: OK, instance_id { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" instance_seqno: 1714115144468685 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:05:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.943527 31219 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.944614 31424 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102" is_pre_election: true
07:05:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.945099 31424 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in term 0.
07:05:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.946209 31236 leader_election.cc:302] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters:
07:05:44.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.946727 31469 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Leader pre-election won for term 1
07:05:44.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.946866 31469 raft_consensus.cc:482] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:05:44.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.946951 31469 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 0 FOLLOWER]: Advancing to term 1
07:05:44.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.949858 31493 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:05:44.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.950174 31469 raft_consensus.cc:504] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.952026 31424 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102"
07:05:44.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.952232 31424 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 0 FOLLOWER]: Advancing to term 1
07:05:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.956115 31424 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in term 1.
07:05:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.956513 31236 leader_election.cc:302] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters:
07:05:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.956539 31469 leader_election.cc:288] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 1 election: Requested vote from peers fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:44.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.956807 31469 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 FOLLOWER]: Leader election won for term 1
07:05:44.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.956957 31469 raft_consensus.cc:686] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 LEADER]: Becoming Leader. State: Replica: 75aa2aca8a1d47b2b09e9bdd5474f9cb, State: Running, Role: LEADER
07:05:44.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.957422 31469 consensus_queue.cc:237] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:44.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.955958 31379 master_runner.cc:186] Error getting master registration for 127.30.122.61:45247: OK, instance_id { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" instance_seqno: 1714115144468685 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:05:44.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.965790 31379 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.966729 31494 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 75aa2aca8a1d47b2b09e9bdd5474f9cb. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.967243 31494 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: LEADER
07:05:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.968283 31496 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:05:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.970461 31496 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:05:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.979089 31424 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Refusing update from remote peer 75aa2aca8a1d47b2b09e9bdd5474f9cb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:05:44.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.979883 31469 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.986184 31481 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 75aa2aca8a1d47b2b09e9bdd5474f9cb. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.986398 31481 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:44.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.990424 31494 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:44.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.990635 31494 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: LEADER
07:05:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.991482 31481 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.991670 31481 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.992223 31496 catalog_manager.cc:1347] Generated new cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.992331 31496 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:05:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:44.996202 31498 mvcc.cc:205] Tried to move back new op lower bound from 7021015633824333824 to 7021015633750368256. Current Snapshot: MvccSnapshot[applied={T|T < 7021015633824333824}]
07:05:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.010612 31496 catalog_manager.cc:1370] Generated new certificate authority record
07:05:45.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.012338 31496 catalog_manager.cc:1504] Loading token signing keys...
07:05:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.026381 31496 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Generated new TSK 0
07:05:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.027504 31496 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.035076 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/logs
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data/info.pb
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.1:0
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.122.1
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.1
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:05:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:05:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:05:45.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.166386 31377 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } attempt: 5
07:05:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.178292 31377 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: Bootstrap starting.
07:05:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.181842 31377 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: Neither blocks nor log segments found. Creating new log.
07:05:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.183023 31377 log.cc:824] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: Log is configured to *not* fsync() on all Append() calls
07:05:45.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.187086 31377 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: No bootstrap required, opened a new log
07:05:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.203799 31377 raft_consensus.cc:348] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.204428 31377 raft_consensus.cc:374] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.204627 31377 raft_consensus.cc:729] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fea5e5cdb708445faa58d8fde854b964, State: Initialized, Role: FOLLOWER
07:05:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.205701 31377 consensus_queue.cc:260] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:45.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.207152 31504 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:45.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.207767 31504 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: FOLLOWER
07:05:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.209434 31341 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fea5e5cdb708445faa58d8fde854b964"
07:05:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.209834 31342 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fea5e5cdb708445faa58d8fde854b964" is_pre_election: true
07:05:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.210847 31340 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 0 FOLLOWER]: Advancing to term 1
07:05:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.211508 31377 sys_catalog.cc:564] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: configured and running, proceeding with master startup.
07:05:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.214306 31340 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 1 FOLLOWER]: Refusing update from remote peer 75aa2aca8a1d47b2b09e9bdd5474f9cb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:05:45.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.215888 31469 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [LEADER]: Connected to new peer: Peer: permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:45.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.257988 31504 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 75aa2aca8a1d47b2b09e9bdd5474f9cb. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:45.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.258289 31504 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: FOLLOWER
07:05:45.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.258672 31511 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:45.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.258968 31511 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: FOLLOWER
07:05:45.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.284760 31520 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.284883 31520 catalog_manager.cc:1552] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: loading cluster ID for follower catalog manager: success
07:05:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.289036 31520 catalog_manager.cc:1574] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: acquiring CA information for follower catalog manager: success
07:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.291502 31520 catalog_manager.cc:1602] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:05:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.325489 31469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.325851 31469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: LEADER
07:05:45.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.424217 31502 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:45.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.424798 31502 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.425122 31502 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:45.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.436689 31502 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:45.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.436991 31502 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:05:45.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.437120 31502 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:05:45.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.437243 31502 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:05:45.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.437393 31502 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.122.1
07:05:45.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.451756 31502 tablet_server_runner.cc:78] Tablet server non-default flags:
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.1:0
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data/info.pb
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.1
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.122.1
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/logs
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:45.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:05:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:45.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.454605 31502 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:45.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.456907 31502 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:45.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.470955 31526 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.472576 31527 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.479377 31529 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:45.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.492545 31502 server_base.cc:1034] running on GCE node
07:05:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.493162 31502 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.494216 31502 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:45.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.495419 31502 hybrid_clock.cc:648] HybridClock initialized: now 1714115145495338 us; error 87 us; skew 500 ppm
07:05:45.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.495720 31502 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.497970 31502 webserver.cc:460] Webserver started at http://127.30.122.1:41023/ using document root <none> and password file <none>
07:05:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.498649 31502 fs_manager.cc:359] Metadata directory not provided
07:05:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.498749 31502 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:45.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.499038 31502 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:05:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.501533 31502 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data/instance:
07:05:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "608d7f45c8734fc880b32194a5ecf4f8"
07:05:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:45 on dist-test-slave-7f9f"
07:05:45.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.502279 31502 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal/instance:
07:05:45.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "608d7f45c8734fc880b32194a5ecf4f8"
07:05:45.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:45 on dist-test-slave-7f9f"
07:05:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.507098 31502 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
07:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.510222 31535 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.511489 31502 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
07:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.511634 31502 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal
07:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "608d7f45c8734fc880b32194a5ecf4f8"
07:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:45 on dist-test-slave-7f9f"
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.511832 31502 fs_report.cc:383] FS layout report
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data/data
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:45.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.531567 31502 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:45.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.532358 31502 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:45.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.532619 31502 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:45.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.535722 31502 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:05:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.535936 31502 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:05:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.536172 31502 ts_tablet_manager.cc:590] Registered 0 tablets
07:05:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.536317 31502 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:05:45.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.637544 31502 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.1:40595
07:05:45.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.638780 31502 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/data/info.pb
07:05:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.639235 31659 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.1:40595 every 8 connection(s)
07:05:45.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.641193 31661 heartbeater.cc:347] Connected to a master server at 127.30.122.62:42411
07:05:45.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.641697 31661 heartbeater.cc:464] Registering TS with master...
07:05:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.643097 31661 heartbeater.cc:511] Master 127.30.122.62:42411 requested a full tablet report, sending...
07:05:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.645080 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 31502
07:05:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.645298 31208 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-0/wal/instance
07:05:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.645332 31250 ts_manager.cc:194] Registered new tserver with Master: 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1:40595)
07:05:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.647408 31250 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.122.1:46829
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.651057 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/logs
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data/info.pb
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.2:0
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.122.2
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.2
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:05:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:05:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:05:45.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.665443 31660 heartbeater.cc:347] Connected to a master server at 127.30.122.60:37495
07:05:45.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.665602 31660 heartbeater.cc:464] Registering TS with master...
07:05:45.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.665943 31660 heartbeater.cc:511] Master 127.30.122.60:37495 requested a full tablet report, sending...
07:05:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.667428 31414 ts_manager.cc:194] Registered new tserver with Master: 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1:40595)
07:05:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.669061 31662 heartbeater.cc:347] Connected to a master server at 127.30.122.61:45247
07:05:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.669198 31662 heartbeater.cc:464] Registering TS with master...
07:05:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.669463 31662 heartbeater.cc:511] Master 127.30.122.61:45247 requested a full tablet report, sending...
07:05:45.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.671164 31332 ts_manager.cc:194] Registered new tserver with Master: 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1:40595)
07:05:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.954864 31493 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.955001 31493 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: loading cluster ID for follower catalog manager: success
07:05:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.959216 31493 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: acquiring CA information for follower catalog manager: success
07:05:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:45.961798 31493 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:05:45.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.996627 31663 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:45.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.996905 31663 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:45.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:45.997064 31663 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.004245 31663 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.004400 31663 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:05:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.004454 31663 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:05:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.004519 31663 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:05:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.004637 31663 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.122.2
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.013442 31663 tablet_server_runner.cc:78] Tablet server non-default flags:
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.2:0
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data/info.pb
07:05:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.2
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.122.2
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/logs
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.014832 31663 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:46.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.016654 31663 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.029376 31673 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.029695 31675 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.030016 31672 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.030352 31663 server_base.cc:1034] running on GCE node
07:05:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.031250 31663 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.032289 31663 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:46.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.033492 31663 hybrid_clock.cc:648] HybridClock initialized: now 1714115146033439 us; error 51 us; skew 500 ppm
07:05:46.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.033752 31663 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:46.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.035863 31663 webserver.cc:460] Webserver started at http://127.30.122.2:45223/ using document root <none> and password file <none>
07:05:46.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.036574 31663 fs_manager.cc:359] Metadata directory not provided
07:05:46.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.036659 31663 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:46.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.036906 31663 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:05:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.039423 31663 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data/instance:
07:05:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d19dc311d24345f39fd9b99f4b936983"
07:05:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:46 on dist-test-slave-7f9f"
07:05:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.040143 31663 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal/instance:
07:05:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d19dc311d24345f39fd9b99f4b936983"
07:05:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:46 on dist-test-slave-7f9f"
07:05:46.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.045289 31663 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
07:05:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.048409 31681 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.049715 31663 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.049868 31663 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d19dc311d24345f39fd9b99f4b936983"
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:46 on dist-test-slave-7f9f"
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.050055 31663 fs_report.cc:383] FS layout report
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data/data
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:46.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.068934 31663 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:46.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.069654 31663 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.069918 31663 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:46.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.072695 31663 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:05:46.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.072798 31663 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:05:46.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.072923 31663 ts_tablet_manager.cc:590] Registered 0 tablets
07:05:46.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.073105 31663 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
07:05:46.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.163954 31663 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.2:42837
07:05:46.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.165408 31663 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/data/info.pb
07:05:46.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.163980 31802 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.2:42837 every 8 connection(s)
07:05:46.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.168596 31804 heartbeater.cc:347] Connected to a master server at 127.30.122.62:42411
07:05:46.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.168892 31804 heartbeater.cc:464] Registering TS with master...
07:05:46.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.169997 31804 heartbeater.cc:511] Master 127.30.122.62:42411 requested a full tablet report, sending...
07:05:46.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.171716 31250 ts_manager.cc:194] Registered new tserver with Master: d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:46.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.172926 31250 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.122.2:45153
07:05:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.174139 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 31663
07:05:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.174310 31208 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-1/wal/instance
07:05:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.178452 31803 heartbeater.cc:347] Connected to a master server at 127.30.122.60:37495
07:05:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.178586 31803 heartbeater.cc:464] Registering TS with master...
07:05:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.179096 31803 heartbeater.cc:511] Master 127.30.122.60:37495 requested a full tablet report, sending...
07:05:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.180119 31414 ts_manager.cc:194] Registered new tserver with Master: d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.181453 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/logs
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data/info.pb
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.3:0
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.122.3
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.3
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:05:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.186987 31805 heartbeater.cc:347] Connected to a master server at 127.30.122.61:45247
07:05:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.187131 31805 heartbeater.cc:464] Registering TS with master...
07:05:46.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.187943 31805 heartbeater.cc:511] Master 127.30.122.61:45247 requested a full tablet report, sending...
07:05:46.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.189148 31332 ts_manager.cc:194] Registered new tserver with Master: d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:46.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.491851 31808 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:46.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.492147 31808 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:46.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.492305 31808 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.499372 31808 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.499523 31808 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:05:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.499577 31808 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:05:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.499639 31808 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:05:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.499758 31808 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.122.3
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.508898 31808 tablet_server_runner.cc:78] Tablet server non-default flags:
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.3:0
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data/info.pb
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.3
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.122.3
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/logs
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.510358 31808 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:46.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.512005 31808 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:46.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.523124 31813 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:46.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.523375 31816 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.523844 31808 server_base.cc:1034] running on GCE node
07:05:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.524091 31814 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.524685 31808 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:46.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.525509 31808 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:46.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.526686 31808 hybrid_clock.cc:648] HybridClock initialized: now 1714115146526642 us; error 38 us; skew 500 ppm
07:05:46.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.526999 31808 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.529031 31808 webserver.cc:460] Webserver started at http://127.30.122.3:33431/ using document root <none> and password file <none>
07:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.529654 31808 fs_manager.cc:359] Metadata directory not provided
07:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.529734 31808 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.529980 31808 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:05:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.532487 31808 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data/instance:
07:05:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8de35652c00d48feb4850f9c3cb5a27f"
07:05:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:46 on dist-test-slave-7f9f"
07:05:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.533164 31808 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal/instance:
07:05:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8de35652c00d48feb4850f9c3cb5a27f"
07:05:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:46 on dist-test-slave-7f9f"
07:05:46.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.537891 31808 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
07:05:46.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.540998 31822 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:46.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.542282 31808 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
07:05:46.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.542441 31808 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal
07:05:46.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8de35652c00d48feb4850f9c3cb5a27f"
07:05:46.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:46 on dist-test-slave-7f9f"
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.542618 31808 fs_report.cc:383] FS layout report
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data/data
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:46.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.557904 31808 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:46.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.558662 31808 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:46.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.558897 31808 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.561539 31808 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:05:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.561635 31808 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:05:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.561743 31808 ts_tablet_manager.cc:590] Registered 0 tablets
07:05:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.561810 31808 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:05:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.616395 31808 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.3:32983
07:05:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.616420 31945 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.3:32983 every 8 connection(s)
07:05:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.617658 31808 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/data/info.pb
07:05:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.618065 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 31808
07:05:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.618268 31208 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/ts-2/wal/instance
07:05:46.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.621001 31947 heartbeater.cc:347] Connected to a master server at 127.30.122.62:42411
07:05:46.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.621243 31947 heartbeater.cc:464] Registering TS with master...
07:05:46.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.622260 31947 heartbeater.cc:511] Master 127.30.122.62:42411 requested a full tablet report, sending...
07:05:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.623708 31250 ts_manager.cc:194] Registered new tserver with Master: 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.624897 31250 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.122.3:33763
07:05:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.629462 31946 heartbeater.cc:347] Connected to a master server at 127.30.122.60:37495
07:05:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.629642 31946 heartbeater.cc:464] Registering TS with master...
07:05:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.630020 31946 heartbeater.cc:511] Master 127.30.122.60:37495 requested a full tablet report, sending...
07:05:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.631124 31414 ts_manager.cc:194] Registered new tserver with Master: 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.634130 31948 heartbeater.cc:347] Connected to a master server at 127.30.122.61:45247
07:05:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.634239 31948 heartbeater.cc:464] Registering TS with master...
07:05:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.634439 31948 heartbeater.cc:511] Master 127.30.122.61:45247 requested a full tablet report, sending...
07:05:46.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.635246 31332 ts_manager.cc:194] Registered new tserver with Master: 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:46.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.642701 31208 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:05:46.644 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:05:46.644 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:05:46.645 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.122.62"
port: 42411
}
bound_http_address {
host: "127.30.122.62"
port: 45449
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.122.61"
port: 45247
}
bound_http_address {
host: "127.30.122.61"
port: 32891
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.122.60"
port: 37495
}
bound_http_address {
host: "127.30.122.60"
port: 38757
}
}
}
07:05:46.646 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:05:46.646 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.30.122.1"
port: 40595
}
bound_http_address {
host: "127.30.122.1"
port: 41023
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.30.122.2"
port: 42837
}
bound_http_address {
host: "127.30.122.2"
port: 45223
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.30.122.3"
port: 32983
}
bound_http_address {
host: "127.30.122.3"
port: 33431
}
}
}
07:05:46.646 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:05:46.648 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:05:46.650 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:46.651 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.122.62:42411(127.30.122.62:42411)] connecting to peer
07:05:46.651 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:46.651 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.122.61:45247(127.30.122.61:45247)] Successfully connected to peer
07:05:46.652 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.122.62:42411(127.30.122.62:42411)] Successfully connected to peer
07:05:46.654 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.122.61
07:05:46.654 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.122.62
07:05:46.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.656417 31661 heartbeater.cc:503] Master 127.30.122.62:42411 was elected leader, sending a full tablet report...
07:05:46.659 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.122.60:37495(127.30.122.60:37495)] Successfully connected to peer
07:05:46.660 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.122.60
07:05:46.666 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb1c9a90a, L:/127.0.0.1:41148 - R:/127.30.122.62:42411] using SASL/PLAIN
07:05:46.669 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:05:46.669 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=20), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [18ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, deferred=null) after lookup
07:05:46.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.671463 31249 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41148:
07:05:46.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
07:05:46.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:05:46.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.673739 31249 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:05:46.692 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe32d021d, L:/127.0.0.1:33522 - R:/127.30.122.60:37495] using SASL/PLAIN
07:05:46.694 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x184693b6, L:/127.0.0.1:37194 - R:/127.30.122.61:45247] using SASL/PLAIN
07:05:46.710 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 7a7e12f7a46a40ee9863f028bde605c0
07:05:46.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.747406 31594 tablet_service.cc:1467] Processing CreateTablet for tablet bd855767784149e3b544eb1adaf714d0 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:05:46.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.748101 31734 tablet_service.cc:1467] Processing CreateTablet for tablet bd855767784149e3b544eb1adaf714d0 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:05:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.749281 31730 tablet_service.cc:1467] Processing CreateTablet for tablet 4ff786b793784c1384710916b2184a07 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:05:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.748908 31875 tablet_service.cc:1467] Processing CreateTablet for tablet bd855767784149e3b544eb1adaf714d0 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:05:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.749424 31594 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd855767784149e3b544eb1adaf714d0. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.750174 31875 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd855767784149e3b544eb1adaf714d0. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.750175 31734 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd855767784149e3b544eb1adaf714d0. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.750715 31730 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ff786b793784c1384710916b2184a07. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.748466 31593 tablet_service.cc:1467] Processing CreateTablet for tablet 4ff786b793784c1384710916b2184a07 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:05:46.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.752131 31874 tablet_service.cc:1467] Processing CreateTablet for tablet 4ff786b793784c1384710916b2184a07 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:05:46.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.752321 31593 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ff786b793784c1384710916b2184a07. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.752519 31874 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ff786b793784c1384710916b2184a07. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.764611 31972 tablet_bootstrap.cc:492] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: Bootstrap starting.
07:05:46.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.767091 31972 tablet_bootstrap.cc:654] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: Neither blocks nor log segments found. Creating new log.
07:05:46.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.768039 31972 log.cc:824] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: Log is configured to *not* fsync() on all Append() calls
07:05:46.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.770722 31971 tablet_bootstrap.cc:492] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: Bootstrap starting.
07:05:46.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.771665 31973 tablet_bootstrap.cc:492] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: Bootstrap starting.
07:05:46.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.772694 31972 tablet_bootstrap.cc:492] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: No bootstrap required, opened a new log
07:05:46.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.772943 31972 ts_tablet_manager.cc:1367] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.002s
07:05:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.774173 31973 tablet_bootstrap.cc:654] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: Neither blocks nor log segments found. Creating new log.
07:05:46.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.775367 31973 log.cc:824] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: Log is configured to *not* fsync() on all Append() calls
07:05:46.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.776865 31971 tablet_bootstrap.cc:654] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: Neither blocks nor log segments found. Creating new log.
07:05:46.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.777992 31973 tablet_bootstrap.cc:492] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: No bootstrap required, opened a new log
07:05:46.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.777992 31971 log.cc:824] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: Log is configured to *not* fsync() on all Append() calls
07:05:46.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.778241 31973 ts_tablet_manager.cc:1367] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.005s
07:05:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.779202 31972 raft_consensus.cc:348] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.779498 31972 raft_consensus.cc:374] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.779587 31972 raft_consensus.cc:729] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d19dc311d24345f39fd9b99f4b936983, State: Initialized, Role: FOLLOWER
07:05:46.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.780270 31972 consensus_queue.cc:260] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.781348 31804 heartbeater.cc:503] Master 127.30.122.62:42411 was elected leader, sending a full tablet report...
07:05:46.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.782141 31971 tablet_bootstrap.cc:492] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: No bootstrap required, opened a new log
07:05:46.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.782490 31971 ts_tablet_manager.cc:1367] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: Time spent bootstrapping tablet: real 0.012s user 0.008s sys 0.000s
07:05:46.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.782908 31972 ts_tablet_manager.cc:1398] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: Time spent starting tablet: real 0.010s user 0.008s sys 0.001s
07:05:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.788259 31973 raft_consensus.cc:348] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.788738 31973 raft_consensus.cc:374] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.788848 31972 tablet_bootstrap.cc:492] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983: Bootstrap starting.
07:05:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.788996 31973 raft_consensus.cc:729] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8de35652c00d48feb4850f9c3cb5a27f, State: Initialized, Role: FOLLOWER
07:05:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.789794 31973 consensus_queue.cc:260] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.791513 31971 raft_consensus.cc:348] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.792033 31971 raft_consensus.cc:374] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.792232 31971 raft_consensus.cc:729] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 608d7f45c8734fc880b32194a5ecf4f8, State: Initialized, Role: FOLLOWER
07:05:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.792834 31972 tablet_bootstrap.cc:654] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983: Neither blocks nor log segments found. Creating new log.
07:05:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.792846 31973 ts_tablet_manager.cc:1398] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: Time spent starting tablet: real 0.014s user 0.011s sys 0.001s
07:05:46.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.793119 31947 heartbeater.cc:503] Master 127.30.122.62:42411 was elected leader, sending a full tablet report...
07:05:46.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.793211 31973 tablet_bootstrap.cc:492] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f: Bootstrap starting.
07:05:46.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.792928 31971 consensus_queue.cc:260] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.794705 31973 tablet_bootstrap.cc:654] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f: Neither blocks nor log segments found. Creating new log.
07:05:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.797638 31971 ts_tablet_manager.cc:1398] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: Time spent starting tablet: real 0.015s user 0.005s sys 0.007s
07:05:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.799147 31973 tablet_bootstrap.cc:492] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f: No bootstrap required, opened a new log
07:05:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.799427 31973 ts_tablet_manager.cc:1367] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.005s
07:05:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.800313 31973 raft_consensus.cc:348] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.800676 31973 raft_consensus.cc:374] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.800846 31973 raft_consensus.cc:729] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8de35652c00d48feb4850f9c3cb5a27f, State: Initialized, Role: FOLLOWER
07:05:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.801153 31973 consensus_queue.cc:260] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.801481 31971 tablet_bootstrap.cc:492] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8: Bootstrap starting.
07:05:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.803206 31971 tablet_bootstrap.cc:654] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8: Neither blocks nor log segments found. Creating new log.
07:05:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.803493 31973 ts_tablet_manager.cc:1398] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
07:05:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809176 31971 tablet_bootstrap.cc:492] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8: No bootstrap required, opened a new log
07:05:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809290 31971 ts_tablet_manager.cc:1367] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.001s
07:05:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809587 31972 tablet_bootstrap.cc:492] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983: No bootstrap required, opened a new log
07:05:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809724 31972 ts_tablet_manager.cc:1367] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983: Time spent bootstrapping tablet: real 0.021s user 0.004s sys 0.009s
07:05:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809780 31971 raft_consensus.cc:348] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809926 31971 raft_consensus.cc:374] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.809980 31971 raft_consensus.cc:729] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 608d7f45c8734fc880b32194a5ecf4f8, State: Initialized, Role: FOLLOWER
07:05:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.810160 31971 consensus_queue.cc:260] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.810386 31972 raft_consensus.cc:348] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.810590 31971 ts_tablet_manager.cc:1398] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
07:05:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.811139 31972 raft_consensus.cc:374] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.811329 31972 raft_consensus.cc:729] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d19dc311d24345f39fd9b99f4b936983, State: Initialized, Role: FOLLOWER
07:05:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.811656 31972 consensus_queue.cc:260] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.812367 31972 ts_tablet_manager.cc:1398] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
07:05:46.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.816306 31979 raft_consensus.cc:482] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:46.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.816498 31979 raft_consensus.cc:504] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.817783 31983 raft_consensus.cc:482] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:46.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.817945 31983 raft_consensus.cc:504] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.818202 31979 leader_election.cc:288] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983), d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:46.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.820356 31983 leader_election.cc:288] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837), 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.825263 31758 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd855767784149e3b544eb1adaf714d0" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d19dc311d24345f39fd9b99f4b936983" is_pre_election: true
07:05:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.825301 31757 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ff786b793784c1384710916b2184a07" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d19dc311d24345f39fd9b99f4b936983" is_pre_election: true
07:05:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.825745 31758 raft_consensus.cc:2454] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 0.
07:05:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.825750 31757 raft_consensus.cc:2454] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 0.
07:05:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.825726 31897 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd855767784149e3b544eb1adaf714d0" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8de35652c00d48feb4850f9c3cb5a27f" is_pre_election: true
07:05:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.826193 31897 raft_consensus.cc:2454] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 0.
07:05:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.826285 31536 leader_election.cc:302] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 608d7f45c8734fc880b32194a5ecf4f8, d19dc311d24345f39fd9b99f4b936983; no voters:
07:05:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.826579 31896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ff786b793784c1384710916b2184a07" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8de35652c00d48feb4850f9c3cb5a27f" is_pre_election: true
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.826676 31536 leader_election.cc:302] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 608d7f45c8734fc880b32194a5ecf4f8, d19dc311d24345f39fd9b99f4b936983; no voters:
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.826802 31896 raft_consensus.cc:2454] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 0.
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.827073 31983 raft_consensus.cc:2781] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.827220 31979 raft_consensus.cc:2781] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.827246 31983 raft_consensus.cc:482] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.827353 31983 raft_consensus.cc:3037] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Advancing to term 1
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.827356 31979 raft_consensus.cc:482] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.827545 31979 raft_consensus.cc:3037] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Advancing to term 1
07:05:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.829484 31983 raft_consensus.cc:504] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.829560 31979 raft_consensus.cc:504] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.829998 31983 leader_election.cc:288] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 election: Requested vote from peers 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983), d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.830163 31979 leader_election.cc:288] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 election: Requested vote from peers d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837), 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.830354 31896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd855767784149e3b544eb1adaf714d0" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8de35652c00d48feb4850f9c3cb5a27f"
07:05:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.830509 31896 raft_consensus.cc:3037] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Advancing to term 1
07:05:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.830917 31757 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ff786b793784c1384710916b2184a07" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d19dc311d24345f39fd9b99f4b936983"
07:05:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.831061 31758 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd855767784149e3b544eb1adaf714d0" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d19dc311d24345f39fd9b99f4b936983"
07:05:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.831246 31758 raft_consensus.cc:3037] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Advancing to term 1
07:05:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.832798 31896 raft_consensus.cc:2454] T bd855767784149e3b544eb1adaf714d0 P 8de35652c00d48feb4850f9c3cb5a27f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 1.
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.833135 31537 leader_election.cc:302] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 608d7f45c8734fc880b32194a5ecf4f8, 8de35652c00d48feb4850f9c3cb5a27f; no voters:
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.833181 31758 raft_consensus.cc:2454] T bd855767784149e3b544eb1adaf714d0 P d19dc311d24345f39fd9b99f4b936983 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 1.
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.833324 31896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ff786b793784c1384710916b2184a07" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8de35652c00d48feb4850f9c3cb5a27f"
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.831095 31757 raft_consensus.cc:3037] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Advancing to term 1
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.833465 31979 raft_consensus.cc:2781] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 FOLLOWER]: Leader election won for term 1
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.833581 31896 raft_consensus.cc:3037] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Advancing to term 1
07:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.833621 31979 raft_consensus.cc:686] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 LEADER]: Becoming Leader. State: Replica: 608d7f45c8734fc880b32194a5ecf4f8, State: Running, Role: LEADER
07:05:46.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.834075 31979 consensus_queue.cc:237] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } }
07:05:46.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.836112 31896 raft_consensus.cc:2454] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 1.
07:05:46.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.836174 31757 raft_consensus.cc:2454] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 1.
07:05:46.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.836552 31536 leader_election.cc:302] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 608d7f45c8734fc880b32194a5ecf4f8, d19dc311d24345f39fd9b99f4b936983; no voters:
07:05:46.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.837028 31983 raft_consensus.cc:2781] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 FOLLOWER]: Leader election won for term 1
07:05:46.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.838311 31250 catalog_manager.cc:5484] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 reported cstate change: term changed from 0 to 1, leader changed from <none> to 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1). New cstate: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } health_report { overall_health: UNKNOWN } } }
07:05:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.840425 31983 raft_consensus.cc:686] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 LEADER]: Becoming Leader. State: Replica: 608d7f45c8734fc880b32194a5ecf4f8, State: Running, Role: LEADER
07:05:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.840696 31983 consensus_queue.cc:237] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.850525 31250 catalog_manager.cc:5484] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 reported cstate change: term changed from 0 to 1, leader changed from <none> to 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1). New cstate: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } health_report { overall_health: UNKNOWN } } }
07:05:46.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.880818 31949 tablet.cc:2369] T 4ff786b793784c1384710916b2184a07 P 8de35652c00d48feb4850f9c3cb5a27f: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.892959 31250 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41162:
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:05:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.893890 31250 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:05:46.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.903751 31730 tablet_service.cc:1467] Processing CreateTablet for tablet 91583874abd345379149bc88532892ee (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:05:46.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.904732 31730 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91583874abd345379149bc88532892ee. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.908577 31874 tablet_service.cc:1467] Processing CreateTablet for tablet 91583874abd345379149bc88532892ee (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:05:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.908557 31593 tablet_service.cc:1467] Processing CreateTablet for tablet 91583874abd345379149bc88532892ee (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:05:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.909255 31874 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91583874abd345379149bc88532892ee. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.909351 31593 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91583874abd345379149bc88532892ee. 1 dirs total, 0 dirs full, 0 dirs failed
07:05:46.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.911578 31972 tablet_bootstrap.cc:492] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: Bootstrap starting.
07:05:46.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.913146 31972 tablet_bootstrap.cc:654] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: Neither blocks nor log segments found. Creating new log.
07:05:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.914968 31973 tablet_bootstrap.cc:492] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: Bootstrap starting.
07:05:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.914958 31664 tablet.cc:2369] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:05:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.915124 31972 tablet_bootstrap.cc:492] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: No bootstrap required, opened a new log
07:05:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.915259 31972 ts_tablet_manager.cc:1367] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
07:05:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.915932 31972 raft_consensus.cc:348] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.916180 31972 raft_consensus.cc:374] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.916231 31971 tablet_bootstrap.cc:492] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: Bootstrap starting.
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.916247 31972 raft_consensus.cc:729] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d19dc311d24345f39fd9b99f4b936983, State: Initialized, Role: FOLLOWER
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.916501 31973 tablet_bootstrap.cc:654] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: Neither blocks nor log segments found. Creating new log.
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.916499 31972 consensus_queue.cc:260] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.916975 31977 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.917330 31977 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.917799 31971 tablet_bootstrap.cc:654] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: Neither blocks nor log segments found. Creating new log.
07:05:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.917786 31972 ts_tablet_manager.cc:1398] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
07:05:46.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.922933 31971 tablet_bootstrap.cc:492] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: No bootstrap required, opened a new log
07:05:46.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.923112 31971 ts_tablet_manager.cc:1367] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
07:05:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.923849 31971 raft_consensus.cc:348] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.924127 31971 raft_consensus.cc:374] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.924206 31971 raft_consensus.cc:729] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 608d7f45c8734fc880b32194a5ecf4f8, State: Initialized, Role: FOLLOWER
07:05:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.924490 31971 consensus_queue.cc:260] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.925048 31971 ts_tablet_manager.cc:1398] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
07:05:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.925014 31983 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.925354 31983 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.929749 31973 tablet_bootstrap.cc:492] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: No bootstrap required, opened a new log
07:05:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.929876 31973 ts_tablet_manager.cc:1367] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.010s
07:05:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.930435 31973 raft_consensus.cc:348] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.930622 31973 raft_consensus.cc:374] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.930670 31973 raft_consensus.cc:729] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8de35652c00d48feb4850f9c3cb5a27f, State: Initialized, Role: FOLLOWER
07:05:46.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.930866 31973 consensus_queue.cc:260] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:46.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.931318 31973 ts_tablet_manager.cc:1398] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
07:05:46.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.931299 31978 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:46.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:46.931671 31978 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:46.941903 31806 tablet.cc:2369] T 4ff786b793784c1384710916b2184a07 P d19dc311d24345f39fd9b99f4b936983: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:05:47.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.028685 31983 raft_consensus.cc:482] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:47.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.028875 31983 raft_consensus.cc:504] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:47.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.029443 31983 leader_election.cc:288] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837), 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.029922 31757 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91583874abd345379149bc88532892ee" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d19dc311d24345f39fd9b99f4b936983" is_pre_election: true
07:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.029995 31896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91583874abd345379149bc88532892ee" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8de35652c00d48feb4850f9c3cb5a27f" is_pre_election: true
07:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.030148 31757 raft_consensus.cc:2454] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 0.
07:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.030212 31896 raft_consensus.cc:2454] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 0.
07:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.030542 31536 leader_election.cc:302] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 608d7f45c8734fc880b32194a5ecf4f8, d19dc311d24345f39fd9b99f4b936983; no voters:
07:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.030854 31983 raft_consensus.cc:2781] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:05:47.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.030936 31983 raft_consensus.cc:482] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:05:47.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.031002 31983 raft_consensus.cc:3037] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 0 FOLLOWER]: Advancing to term 1
07:05:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.032866 31983 raft_consensus.cc:504] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.033354 31983 leader_election.cc:288] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 election: Requested vote from peers d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837), 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.033685 31757 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91583874abd345379149bc88532892ee" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d19dc311d24345f39fd9b99f4b936983"
07:05:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.033856 31757 raft_consensus.cc:3037] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 0 FOLLOWER]: Advancing to term 1
07:05:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.033841 31896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91583874abd345379149bc88532892ee" candidate_uuid: "608d7f45c8734fc880b32194a5ecf4f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8de35652c00d48feb4850f9c3cb5a27f"
07:05:47.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.034018 31896 raft_consensus.cc:3037] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 0 FOLLOWER]: Advancing to term 1
07:05:47.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.035669 31757 raft_consensus.cc:2454] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 1.
07:05:47.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.035832 31896 raft_consensus.cc:2454] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 608d7f45c8734fc880b32194a5ecf4f8 in term 1.
07:05:47.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.036057 31536 leader_election.cc:302] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 608d7f45c8734fc880b32194a5ecf4f8, d19dc311d24345f39fd9b99f4b936983; no voters:
07:05:47.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.036393 31983 raft_consensus.cc:2781] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 FOLLOWER]: Leader election won for term 1
07:05:47.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.036598 31983 raft_consensus.cc:686] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [term 1 LEADER]: Becoming Leader. State: Replica: 608d7f45c8734fc880b32194a5ecf4f8, State: Running, Role: LEADER
07:05:47.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.036854 31983 consensus_queue.cc:237] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } }
07:05:47.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.038519 31987 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: TxnStatusTablet state changed. Reason: New leader 608d7f45c8734fc880b32194a5ecf4f8. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.038923 31987 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: This TxnStatusTablet replica's current role is: LEADER
07:05:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.039819 31250 catalog_manager.cc:5484] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 reported cstate change: term changed from 0 to 1, leader changed from <none> to 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1). New cstate: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } health_report { overall_health: UNKNOWN } } }
07:05:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.041956 32001 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
07:05:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.042172 32001 txn_status_manager.cc:929] Loading transaction status metadata into memory...
07:05:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.074147 31896 raft_consensus.cc:1261] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f [term 1 FOLLOWER]: Refusing update from remote peer 608d7f45c8734fc880b32194a5ecf4f8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:05:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.075359 31757 raft_consensus.cc:1261] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983 [term 1 FOLLOWER]: Refusing update from remote peer 608d7f45c8734fc880b32194a5ecf4f8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:05:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.075911 31983 consensus_queue.cc:1035] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.076581 31983 consensus_queue.cc:1035] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:47.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.099805 31978 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: TxnStatusTablet state changed. Reason: New leader 608d7f45c8734fc880b32194a5ecf4f8. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.100112 31978 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:47.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.103390 31977 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: TxnStatusTablet state changed. Reason: New leader 608d7f45c8734fc880b32194a5ecf4f8. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.103639 31977 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:47.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.109392 32004 mvcc.cc:205] Tried to move back new op lower bound from 7021015642405031936 to 7021015642264985600. Current Snapshot: MvccSnapshot[applied={T|T < 7021015642405031936}]
07:05:47.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.113584 31987 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.113818 31987 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: This TxnStatusTablet replica's current role is: LEADER
07:05:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.119596 31977 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.119825 31977 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P d19dc311d24345f39fd9b99f4b936983: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:47.121 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
07:05:47.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.123142 31978 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.123356 31978 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 8de35652c00d48feb4850f9c3cb5a27f: This TxnStatusTablet replica's current role is: FOLLOWER
07:05:47.125 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 7a7e12f7a46a40ee9863f028bde605c0
07:05:47.125 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.122.62:42411
07:05:47.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:05:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.128680 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 31219
07:05:47.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.129703 31987 tablet_replica.cc:439] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "608d7f45c8734fc880b32194a5ecf4f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" member_type: VOTER last_known_addr { host: "127.30.122.1" port: 40595 } } peers { permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 } } peers { permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 } } }
07:05:47.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.129937 31987 tablet_replica.cc:442] T 91583874abd345379149bc88532892ee P 608d7f45c8734fc880b32194a5ecf4f8: This TxnStatusTablet replica's current role is: LEADER
07:05:47.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.130115 31291 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:47.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.245064 31979 consensus_queue.cc:1035] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.259135 31983 consensus_queue.cc:1035] T 4ff786b793784c1384710916b2184a07 P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:47.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.327013 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 31219
07:05:47.344 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.122.62:42411(127.30.122.62:42411)] handling channelInactive
07:05:47.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.62:42411(127.30.122.62:42411)] cleaning up while in state READY due to: connection closed
07:05:47.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.345440 31823 connection.cc:535] client connection to 127.30.122.62:42411 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:47.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.345872 31947 heartbeater.cc:644] Failed to heartbeat to 127.30.122.62:42411 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:47.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.346160 31804 heartbeater.cc:644] Failed to heartbeat to 127.30.122.62:42411 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.30.122.62:42411 (error 108)
07:05:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.347029 31536 connection.cc:535] client connection to 127.30.122.62:42411 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:47.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.347522 31661 heartbeater.cc:644] Failed to heartbeat to 127.30.122.62:42411 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.348475 32015 negotiation.cc:337] Failed RPC negotiation. Trace:
07:05:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:47.346689 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.30.122.62:42411 (local address 127.30.122.2:33047)
07:05:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:47.347551 (+ 862us) negotiation.cc:107] Waiting for socket to connect
07:05:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:47.347695 (+ 144us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection reset by peer (error 104)
07:05:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":746,"thread_start_us":1191,"threads_started":1}
07:05:47.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.359155 31983 consensus_queue.cc:1035] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" member_type: VOTER last_known_addr { host: "127.30.122.2" port: 42837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:47.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.363795 31987 consensus_queue.cc:1035] T bd855767784149e3b544eb1adaf714d0 P 608d7f45c8734fc880b32194a5ecf4f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" member_type: VOTER last_known_addr { host: "127.30.122.3" port: 32983 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:05:47.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.626317 32021 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 75aa2aca8a1d47b2b09e9bdd5474f9cb)
07:05:47.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.626498 32021 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.627391 32022 raft_consensus.cc:482] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 75aa2aca8a1d47b2b09e9bdd5474f9cb)
07:05:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.627622 32022 raft_consensus.cc:504] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.627918 32021 leader_election.cc:288] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247)
07:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.629047 31397 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111)
07:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.629053 31340 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a08014ec6614098a83031094f54a102" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fea5e5cdb708445faa58d8fde854b964" is_pre_election: true
07:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.629276 31340 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a08014ec6614098a83031094f54a102 in term 1.
07:05:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.630096 31397 leader_election.cc:302] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a08014ec6614098a83031094f54a102, fea5e5cdb708445faa58d8fde854b964; no voters:
07:05:47.631 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.62:42411(127.30.122.62:42411)] connecting to peer
07:05:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.630841 32021 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Leader pre-election won for term 2
07:05:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.630966 32021 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 75aa2aca8a1d47b2b09e9bdd5474f9cb)
07:05:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.631063 32021 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 1 FOLLOWER]: Advancing to term 2
07:05:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.631136 31397 leader_election.cc:334] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111)
07:05:47.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.62:42411(127.30.122.62:42411)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.632 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.30.122.62:42411(127.30.122.62:42411) for tablet Kudu Master: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.632773 31424 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fea5e5cdb708445faa58d8fde854b964" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102" is_pre_election: true
07:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.633216 32022 leader_election.cc:288] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.633987 32021 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.634415 31424 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fea5e5cdb708445faa58d8fde854b964 in current term 2: Already voted for candidate 6a08014ec6614098a83031094f54a102 in this term.
07:05:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.635159 31316 leader_election.cc:334] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111)
07:05:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.635385 32021 leader_election.cc:288] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 2 election: Requested vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247)
07:05:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.635480 31318 leader_election.cc:302] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fea5e5cdb708445faa58d8fde854b964; no voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb
07:05:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.635938 32022 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 1 FOLLOWER]: Advancing to term 2
07:05:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.636124 31340 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a08014ec6614098a83031094f54a102" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fea5e5cdb708445faa58d8fde854b964"
07:05:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.636514 31397 leader_election.cc:334] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111)
07:05:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.638882 32022 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
07:05:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.641934 31340 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a08014ec6614098a83031094f54a102 in term 2.
07:05:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.642629 31397 leader_election.cc:302] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a08014ec6614098a83031094f54a102, fea5e5cdb708445faa58d8fde854b964; no voters: 75aa2aca8a1d47b2b09e9bdd5474f9cb
07:05:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.643075 32021 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 FOLLOWER]: Leader election won for term 2
07:05:47.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.644255 32021 raft_consensus.cc:686] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 LEADER]: Becoming Leader. State: Replica: 6a08014ec6614098a83031094f54a102, State: Running, Role: LEADER
07:05:47.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.644719 32021 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:47.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.645941 32027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a08014ec6614098a83031094f54a102. Latest consensus state: current_term: 2 leader_uuid: "6a08014ec6614098a83031094f54a102" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:47.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.646209 32027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: LEADER
07:05:47.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.646775 32028 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:05:47.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.649116 32028 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236]
07:05:47.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.649600 32028 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]
07:05:47.650 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.62:42411(127.30.122.62:42411)] connecting to peer
07:05:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.651374 32028 tablet_loader.cc:96] loaded metadata for tablet 4ff786b793784c1384710916b2184a07 (table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0])
07:05:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.651896 32028 tablet_loader.cc:96] loaded metadata for tablet 91583874abd345379149bc88532892ee (table kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236])
07:05:47.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.62:42411(127.30.122.62:42411)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.652312 32028 tablet_loader.cc:96] loaded metadata for tablet bd855767784149e3b544eb1adaf714d0 (table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0])
07:05:47.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.62:42411: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.652623 32028 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:05:47.653 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:47.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.653573 32028 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:47.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.653663 32028 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:05:47.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.654765 32028 catalog_manager.cc:1504] Loading token signing keys...
07:05:47.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.655587 32028 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Loaded TSK: 0
07:05:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.656386 32028 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:05:47.670 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.62:42411(127.30.122.62:42411)] connecting to peer
07:05:47.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.62:42411(127.30.122.62:42411)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.62:42411: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411
07:05:47.673 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:05:47.674 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=43), Traces: [0ms] sending RPC to server master-127.30.122.62:42411, [1ms] received response from server master-127.30.122.62:42411: Network error: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [20ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: Network error: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411, [38ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [40ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: Network error: Failed to connect to peer master-127.30.122.62:42411(127.30.122.62:42411): Connection refused: /127.30.122.62:42411, [40ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, deferred=Deferred@753414888(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@6abffc2 -> (continuation of Deferred@964442165 after retry RPC after error@988090776), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@57770b5e -> (continuation of Deferred@964442165 after retry RPC after error@988090776))) after lookup
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.714948 31413 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33550:
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:05:47.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.717105 31413 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:05:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.735968 31340 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Refusing update from remote peer 6a08014ec6614098a83031094f54a102: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
07:05:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.736616 32027 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
07:05:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.737936 32022 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a08014ec6614098a83031094f54a102. Latest consensus state: current_term: 2 leader_uuid: "6a08014ec6614098a83031094f54a102" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.738230 32022 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: FOLLOWER
07:05:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:47.739256 31397 consensus_peers.cc:485] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 -> Peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Couldn't send request to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:05:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.739581 32021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "6a08014ec6614098a83031094f54a102" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.739857 32021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: LEADER
07:05:47.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.740293 32022 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "6a08014ec6614098a83031094f54a102" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:47.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.740464 32022 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: FOLLOWER
07:05:47.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:47.822916 31660 heartbeater.cc:503] Master 127.30.122.60:37495 was elected leader, sending a full tablet report...
07:05:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:48.180754 31397 consensus_peers.cc:485] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 -> Peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Couldn't send request to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
07:05:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:48.681516 31397 consensus_peers.cc:485] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 -> Peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Couldn't send request to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
07:05:49.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.234283 31397 consensus_peers.cc:485] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 -> Peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Couldn't send request to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
07:05:49.352 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet bd855767784149e3b544eb1adaf714d0 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
07:05:49.352 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 4ff786b793784c1384710916b2184a07 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
07:05:49.352 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=bd855767784149e3b544eb1adaf714d0@[608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)[L],8de35652c00d48feb4850f9c3cb5a27f(127.30.122.3:32983),d19dc311d24345f39fd9b99f4b936983(127.30.122.2:42837)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=4ff786b793784c1384710916b2184a07@[608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)[L],8de35652c00d48feb4850f9c3cb5a27f(127.30.122.3:32983),d19dc311d24345f39fd9b99f4b936983(127.30.122.2:42837)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
07:05:49.353 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.122.60:37495, [2ms] Sub RPC GetTableLocations: received response from server master-127.30.122.60:37495: OK, deferred=null) after lookup
07:05:49.353 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)] connecting to peer
07:05:49.354 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)] Successfully connected to peer
07:05:49.364 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xbc985406, L:/127.0.0.1:42620 - R:/127.30.122.1:40595] using TOKEN/null
07:05:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.367823 31563 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.30.122.62:42411)
07:05:49.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.369892 31803 heartbeater.cc:503] Master 127.30.122.60:37495 was elected leader, sending a full tablet report...
07:05:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.372974 31946 heartbeater.cc:503] Master 127.30.122.60:37495 was elected leader, sending a full tablet report...
07:05:49.402 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.122.62:42411
07:05:49.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:05:49.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.403199 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/logs
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/info.pb
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.62:42411
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.62
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45449
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:05:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:05:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.710391 32066 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.710657 32066 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.710744 32066 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.717823 32066 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.717912 32066 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:05:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.717952 32066 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.717998 32066 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.718034 32066 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.718080 32066 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.718123 32066 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
07:05:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.726697 32066 master_runner.cc:386] Master server non-default flags:
07:05:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.62:42411
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/info.pb
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.62
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45449
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/logs
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.727839 32066 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.729492 32066 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:49.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.739287 32073 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:49.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.739668 32075 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.740171 32072 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.740542 32066 server_base.cc:1034] running on GCE node
07:05:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.741262 32066 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.742707 32066 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.743896 32066 hybrid_clock.cc:648] HybridClock initialized: now 1714115149743851 us; error 37 us; skew 500 ppm
07:05:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.744217 32066 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:49.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.746287 32066 webserver.cc:460] Webserver started at http://127.30.122.62:45449/ using document root <none> and password file <none>
07:05:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.746994 32066 fs_manager.cc:359] Metadata directory not provided
07:05:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.747089 32066 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:49.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.751945 32066 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
07:05:49.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.754480 32081 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.755647 32066 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.755807 32066 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb"
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.756572 32066 fs_report.cc:383] FS layout report
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/wal
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/data
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:49.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.772694 32066 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:49.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.773417 32066 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:49.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.773658 32066 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:49.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.776749 31397 consensus_peers.cc:485] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 -> Peer 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411): Couldn't send request to peer 75aa2aca8a1d47b2b09e9bdd5474f9cb. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.62:42411: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
07:05:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.793085 32066 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.62:42411
07:05:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.793125 32142 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.62:42411 every 8 connection(s)
07:05:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.794467 32066 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-0/data/info.pb
07:05:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.796628 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 32066
07:05:49.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:49.797 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.122.60:37495
07:05:49.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:05:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.797636 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 31379
07:05:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.798722 31454 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.802028 32143 sys_catalog.cc:263] Verifying existing consensus state
07:05:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.805289 32143 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Bootstrap starting.
07:05:49.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.820672 32143 log.cc:824] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Log is configured to *not* fsync() on all Append() calls
07:05:49.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.838173 32143 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:05:49.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.838762 32143 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Bootstrap complete.
07:05:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.849880 31661 heartbeater.cc:347] Connected to a master server at 127.30.122.62:42411
07:05:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.849524 32143 raft_consensus.cc:348] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.850615 32143 raft_consensus.cc:729] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 75aa2aca8a1d47b2b09e9bdd5474f9cb, State: Initialized, Role: FOLLOWER
07:05:49.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.851284 32143 consensus_queue.cc:260] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.852177 32149 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.852840 32149 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: FOLLOWER
07:05:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.853273 32143 sys_catalog.cc:564] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: configured and running, proceeding with master startup.
07:05:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.866999 32160 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.867094 32160 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: loading cluster ID for follower catalog manager: success
07:05:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.871155 32160 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: acquiring CA information for follower catalog manager: success
07:05:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.874125 32160 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:05:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.939710 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 31379
07:05:49.956 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.122.60:37495(127.30.122.60:37495)] handling channelInactive
07:05:49.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.956296 31536 connection.cc:535] server connection from 127.0.0.1:42618 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:49.956 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state READY due to: connection closed
07:05:49.956 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:49.956 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.30.122.60:37495(127.30.122.60:37495) for tablet Kudu Master: connection closed
07:05:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:49.957707 32162 negotiation.cc:337] Failed RPC negotiation. Trace:
07:05:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:49.888503 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.30.122.60:37495 (local address 127.0.0.1:33564)
07:05:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:49.888663 (+ 160us) negotiation.cc:107] Waiting for socket to connect
07:05:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:49.888682 (+ 19us) client_negotiation.cc:174] Beginning negotiation
07:05:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:49.888796 (+ 114us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
07:05:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:05:49.956361 (+ 67565us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":56}
07:05:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.958072 32066 master_runner.cc:186] Error getting master registration for 127.30.122.60:37495: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
07:05:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:49.965683 32066 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:49.970 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.62:42411(127.30.122.62:42411)] connecting to peer
07:05:49.971 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:49.971 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.122.62:42411(127.30.122.62:42411)] Successfully connected to peer
07:05:49.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:49.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:49.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:49.985 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd696c45e, L:/127.0.0.1:41680 - R:/127.30.122.62:42411] using TOKEN/null
07:05:49.987 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:49.990 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:49.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:49.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:49.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:49.993 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:50.010 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:50.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:50.030 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:50.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.031 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:50.070 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.30.122.60:37495, [146ms] received response from server master-127.30.122.60:37495: Network error: connection closed, [146ms] delaying RPC due to: Network error: connection closed, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [162ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [162ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [177ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [181ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [201ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [201ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [202ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [221ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [222ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, deferred=Deferred@600171333(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@2494e895 -> (continuation of Deferred@576933651 after retry RPC after error@478817795) -> (continuation of Deferred@274282780 after retry RPC after error@319466434) -> (continuation of Deferred@732525927 after retry RPC after error@1713290279) -> (continuation of Deferred@1062664225 after retry RPC after error@2042888370), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@19a640a0 -> (continuation of Deferred@576933651 after retry RPC after error@478817795) -> (continuation of Deferred@274282780 after retry RPC after error@319466434) -> (continuation of Deferred@732525927 after retry RPC after error@1713290279) -> (continuation of Deferred@1062664225 after retry RPC after error@2042888370)))
07:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.076154 32165 raft_consensus.cc:482] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 6a08014ec6614098a83031094f54a102)
07:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.076588 32165 raft_consensus.cc:504] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.077581 32165 leader_election.cc:288] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:50.071 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:748)
07:05:50.090 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:50.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.093 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.093 [WARN - pool-49-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:50.093909 31318 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:05:50.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:50.095748 31318 leader_election.cc:334] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.096160 32108 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fea5e5cdb708445faa58d8fde854b964" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" is_pre_election: true
07:05:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.096762 32108 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fea5e5cdb708445faa58d8fde854b964 in term 1.
07:05:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.097230 31316 leader_election.cc:302] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 75aa2aca8a1d47b2b09e9bdd5474f9cb, fea5e5cdb708445faa58d8fde854b964; no voters: 6a08014ec6614098a83031094f54a102
07:05:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.097489 32165 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Leader pre-election won for term 3
07:05:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.097599 32165 raft_consensus.cc:482] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 6a08014ec6614098a83031094f54a102)
07:05:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.097679 32165 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 2 FOLLOWER]: Advancing to term 3
07:05:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.099470 32165 raft_consensus.cc:504] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.100028 32165 leader_election.cc:288] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 3 election: Requested vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.100369 32108 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fea5e5cdb708445faa58d8fde854b964" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb"
07:05:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.100579 32108 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 1 FOLLOWER]: Advancing to term 3
07:05:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:50.101688 31318 leader_election.cc:334] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111)
07:05:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.106391 32108 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fea5e5cdb708445faa58d8fde854b964 in term 3.
07:05:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.106792 31316 leader_election.cc:302] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 75aa2aca8a1d47b2b09e9bdd5474f9cb, fea5e5cdb708445faa58d8fde854b964; no voters: 6a08014ec6614098a83031094f54a102
07:05:50.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.107126 32165 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 3 FOLLOWER]: Leader election won for term 3
07:05:50.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.107578 32165 raft_consensus.cc:686] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [term 3 LEADER]: Becoming Leader. State: Replica: fea5e5cdb708445faa58d8fde854b964, State: Running, Role: LEADER
07:05:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.108155 32165 consensus_queue.cc:237] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.110055 32169 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fea5e5cdb708445faa58d8fde854b964. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:50.110 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:50.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.111150 32169 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: LEADER
07:05:50.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.112315 32171 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:05:50.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.113 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.116446 32171 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236]
07:05:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.117018 32171 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]
07:05:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.118599 32171 tablet_loader.cc:96] loaded metadata for tablet 4ff786b793784c1384710916b2184a07 (table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0])
07:05:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.118988 32171 tablet_loader.cc:96] loaded metadata for tablet 91583874abd345379149bc88532892ee (table kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236])
07:05:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.119328 32171 tablet_loader.cc:96] loaded metadata for tablet bd855767784149e3b544eb1adaf714d0 (table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0])
07:05:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.119606 32171 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:05:50.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.120435 32171 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:50.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.120525 32171 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:05:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.121611 32171 catalog_manager.cc:1504] Loading token signing keys...
07:05:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.122419 32171 catalog_manager.cc:5868] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964: Loaded TSK: 0
07:05:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.123172 32171 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:05:50.130 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:50.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.60:37495: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495
07:05:50.133 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:05:50.134 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=3, TimeoutTracker(timeout=250, elapsed=44), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [4ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [24ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [42ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: Network error: Failed to connect to peer master-127.30.122.60:37495(127.30.122.60:37495): Connection refused: /127.30.122.60:37495, [44ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: OK, deferred=Deferred@1284439762(state=PENDING, result=null, callback=(continuation of Deferred@1529177910 after retry RPC after error@1922654216) -> (continuation of Deferred@970519206 after retry RPC after error@43770657), errback=(continuation of Deferred@1529177910 after retry RPC after error@1922654216) -> (continuation of Deferred@970519206 after retry RPC after error@43770657))) after lookup
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.168862 31332 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37238:
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:50.175402 31332 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:05:50.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:50.191246 31318 consensus_peers.cc:485] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 -> Peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Couldn't send request to peer 6a08014ec6614098a83031094f54a102. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:05:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.203984 32108 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Refusing update from remote peer fea5e5cdb708445faa58d8fde854b964: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
07:05:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.204828 32169 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [LEADER]: Connected to new peer: Peer: permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
07:05:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.208729 32183 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.209134 32183 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: FOLLOWER
07:05:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.209643 32149 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader fea5e5cdb708445faa58d8fde854b964. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.209887 32149 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: FOLLOWER
07:05:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.211410 32165 sys_catalog.cc:455] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.211696 32165 sys_catalog.cc:458] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 [sys.catalog]: This master's current role is: LEADER
07:05:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.212059 32149 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.212265 32149 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: FOLLOWER
07:05:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.384502 31947 heartbeater.cc:347] Connected to a master server at 127.30.122.62:42411
07:05:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.385130 31804 heartbeater.cc:347] Connected to a master server at 127.30.122.62:42411
07:05:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.386785 32098 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" instance_seqno: 1714115146609654) as {username='slave'} at 127.30.122.3:50879; Asking this server to re-register.
07:05:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.386802 32097 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" instance_seqno: 1714115146156363) as {username='slave'} at 127.30.122.2:51193; Asking this server to re-register.
07:05:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.387439 31947 heartbeater.cc:464] Registering TS with master...
07:05:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.387455 31804 heartbeater.cc:464] Registering TS with master...
07:05:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.388888 32097 ts_manager.cc:194] Registered new tserver with Master: d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:50.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.389861 32098 ts_manager.cc:194] Registered new tserver with Master: 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:50.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:50.711710 31318 consensus_peers.cc:485] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 -> Peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Couldn't send request to peer 6a08014ec6614098a83031094f54a102. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
07:05:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.842725 32098 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" instance_seqno: 1714115145629993) as {username='slave'} at 127.30.122.1:47645; Asking this server to re-register.
07:05:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.843381 31661 heartbeater.cc:464] Registering TS with master...
07:05:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.844718 32098 ts_manager.cc:194] Registered new tserver with Master: 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1:40595)
07:05:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:50.845427 31662 heartbeater.cc:503] Master 127.30.122.61:45247 was elected leader, sending a full tablet report...
07:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:51.183414 31318 consensus_peers.cc:485] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 -> Peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Couldn't send request to peer 6a08014ec6614098a83031094f54a102. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
07:05:51.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:51.379242 31805 heartbeater.cc:503] Master 127.30.122.61:45247 was elected leader, sending a full tablet report...
07:05:51.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:51.380148 31948 heartbeater.cc:503] Master 127.30.122.61:45247 was elected leader, sending a full tablet report...
07:05:51.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:51.698361 31318 consensus_peers.cc:485] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 -> Peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Couldn't send request to peer 6a08014ec6614098a83031094f54a102. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
07:05:51.977 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.122.60:37495
07:05:51.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:51.978452 31208 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/logs
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/info.pb
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.60:37495
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.60
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38757
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:05:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:05:52.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.248539 31318 consensus_peers.cc:485] T 00000000000000000000000000000000 P fea5e5cdb708445faa58d8fde854b964 -> Peer 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495): Couldn't send request to peer 6a08014ec6614098a83031094f54a102. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.60:37495: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
07:05:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.285146 32209 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.285420 32209 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.285504 32209 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.292903 32209 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.292996 32209 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.293032 32209 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.293080 32209 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.293116 32209 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.293164 32209 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.293202 32209 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.302208 32209 master_runner.cc:386] Master server non-default flags:
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.122.20:37875
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.122.62:42411,127.30.122.61:45247,127.30.122.60:37495
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.122.60:37495
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/info.pb
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.122.60
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38757
07:05:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/logs
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:05:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.303375 32209 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:52.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.305054 32209 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:05:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.315047 32214 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.315495 32215 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.315861 32209 server_base.cc:1034] running on GCE node
07:05:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.316164 32217 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.316946 32209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.318221 32209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.319414 32209 hybrid_clock.cc:648] HybridClock initialized: now 1714115152319372 us; error 38 us; skew 500 ppm
07:05:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.319695 32209 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:52.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.321728 32209 webserver.cc:460] Webserver started at http://127.30.122.60:38757/ using document root <none> and password file <none>
07:05:52.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.322345 32209 fs_manager.cc:359] Metadata directory not provided
07:05:52.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.322439 32209 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:52.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.327149 32209 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.002s sys 0.000s
07:05:52.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.329707 32223 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.330866 32209 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.331001 32209 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data,/tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a08014ec6614098a83031094f54a102"
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:05:44 on dist-test-slave-7f9f"
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.331702 32209 fs_report.cc:383] FS layout report
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/wal
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/data
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:52.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.377341 32209 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:05:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.378223 32209 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.378466 32209 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.398551 32209 rpc_server.cc:307] RPC server started. Bound to: 127.30.122.60:37495
07:05:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.398595 32286 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.122.60:37495 every 8 connection(s)
07:05:52.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.399925 32209 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIiZZ1l/test-tmp/mini-kudu-cluster8277935296640499872/master-2/data/info.pb
07:05:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.401557 31208 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu as pid 32209
07:05:52.401 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:52.402 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.122.61:45247
07:05:52.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:05:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.402558 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 31300
07:05:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.403610 31372 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.407330 32287 sys_catalog.cc:263] Verifying existing consensus state
07:05:52.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.410310 32287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Bootstrap starting.
07:05:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.425078 32287 log.cc:824] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Log is configured to *not* fsync() on all Append() calls
07:05:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.437942 32287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:05:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.438324 32287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: Bootstrap complete.
07:05:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.445681 32287 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.446409 32287 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a08014ec6614098a83031094f54a102, State: Initialized, Role: FOLLOWER
07:05:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.447049 32287 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.447866 32291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.448235 32291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.448858 32287 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: configured and running, proceeding with master startup.
07:05:52.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.470690 32302 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.470804 32302 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: loading cluster ID for follower catalog manager: success
07:05:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.474754 32302 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: acquiring CA information for follower catalog manager: success
07:05:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.477141 32302 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:05:52.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.556342 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 31300
07:05:52.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.557737 32252 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 2 FOLLOWER]: Advancing to term 3
07:05:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.572721 32311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.573284 32311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.573426 32291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fea5e5cdb708445faa58d8fde854b964. Latest consensus state: current_term: 3 leader_uuid: "fea5e5cdb708445faa58d8fde854b964" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.573680 32291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.579576 32224 connection.cc:535] client connection to 127.30.122.61:45247 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.579917 32209 master_runner.cc:216] Error getting master consensus for 127.30.122.61:45247: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.580235 32209 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.580174 32083 connection.cc:535] server connection from 127.0.0.1:41692 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:52.580 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.122.61:45247(127.30.122.61:45247)] handling channelInactive
07:05:52.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state READY due to: connection closed
07:05:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.580785 32226 connection.cc:535] server connection from 127.0.0.1:34106 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:52.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:52.790 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.791 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.30.122.61:45247(127.30.122.61:45247) for tablet Kudu Master: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.810 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.811 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.60:37495(127.30.122.60:37495)] connecting to peer
07:05:52.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.811 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.122.60:37495(127.30.122.60:37495)] Successfully connected to peer
07:05:52.823 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x58456561, L:/127.0.0.1:34122 - R:/127.30.122.60:37495] using TOKEN/null
07:05:52.825 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:52.830 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.832 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:52.850 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.852612 31660 heartbeater.cc:347] Connected to a master server at 127.30.122.60:37495
07:05:52.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.855367 32242 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "608d7f45c8734fc880b32194a5ecf4f8" instance_seqno: 1714115145629993) as {username='slave'} at 127.30.122.1:54511; Asking this server to re-register.
07:05:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.856042 31660 heartbeater.cc:464] Registering TS with master...
07:05:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.857388 32242 ts_manager.cc:194] Registered new tserver with Master: 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1:40595)
07:05:52.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.859093 32314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fea5e5cdb708445faa58d8fde854b964)
07:05:52.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.859259 32314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.860841 32314 leader_election.cc:288] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.861753 32082 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:52.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.863226 32082 leader_election.cc:334] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.864367 32291 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fea5e5cdb708445faa58d8fde854b964)
07:05:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.864563 32291 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.866418 32291 leader_election.cc:288] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247)
07:05:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.866751 32108 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a08014ec6614098a83031094f54a102" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" is_pre_election: true
07:05:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.866947 32108 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a08014ec6614098a83031094f54a102 in term 3.
07:05:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.867100 32224 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.867218 32252 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102" is_pre_election: true
07:05:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.867667 32252 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in term 3.
07:05:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.867976 32224 leader_election.cc:302] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters:
07:05:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868161 32084 leader_election.cc:302] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters: fea5e5cdb708445faa58d8fde854b964
07:05:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868526 32314 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Leader pre-election won for term 4
07:05:52.870 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.871 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868639 32314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Starting leader election (detected failure of leader fea5e5cdb708445faa58d8fde854b964)
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868758 32314 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 3 FOLLOWER]: Advancing to term 4
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868798 32291 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 3 FOLLOWER]: Leader pre-election won for term 4
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.868865 32224 leader_election.cc:334] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868904 32291 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 3 FOLLOWER]: Starting leader election (detected failure of leader fea5e5cdb708445faa58d8fde854b964)
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.868995 32291 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 3 FOLLOWER]: Advancing to term 4
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.871590 32291 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.871590 32314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:52.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.872774 32314 leader_election.cc:288] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 4 election: Requested vote from peers fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:52.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.873061 32108 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a08014ec6614098a83031094f54a102" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb"
07:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.873304 32108 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Leader election vote request: Denying vote to candidate 6a08014ec6614098a83031094f54a102 in current term 4: Already voted for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in this term.
07:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.873466 32082 leader_election.cc:334] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.873710 32252 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102"
07:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.873994 32252 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Leader election vote request: Denying vote to candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in current term 4: Already voted for candidate 6a08014ec6614098a83031094f54a102 in this term.
07:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.874259 32291 leader_election.cc:288] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 4 election: Requested vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247)
07:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.874656 32084 leader_election.cc:302] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 4 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters: 6a08014ec6614098a83031094f54a102, fea5e5cdb708445faa58d8fde854b964
07:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:52.874972 32224 leader_election.cc:334] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.875083 32224 leader_election.cc:302] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 4 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a08014ec6614098a83031094f54a102; no voters: 75aa2aca8a1d47b2b09e9bdd5474f9cb, fea5e5cdb708445faa58d8fde854b964
07:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.875211 32314 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Leader election lost for term 4. Reason: could not achieve majority
07:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:52.875394 32291 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Leader election lost for term 4. Reason: could not achieve majority
07:05:52.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.892 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:52.910 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:52.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:52.912 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.010 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.013 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.30.122.61:45247, [1ms] received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [21ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [42ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [63ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [82ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [101ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [102ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [121ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [122ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [222ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [222ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, deferred=Deferred@142150689(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@2bccb6b2 -> (continuation of Deferred@1286901281 after retry RPC after error@856628109) -> (continuation of Deferred@1935081929 after retry RPC after error@1038110775) -> (continuation of Deferred@1439499308 after retry RPC after error@508931691) -> (continuation of Deferred@1439153441 after retry RPC after error@1780035142) -> (continuation of Deferred@1472768845 after retry RPC after error@1683845417) -> (continuation of Deferred@211547702 after retry RPC after error@794002173), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@3dd3c699 -> (continuation of Deferred@1286901281 after retry RPC after error@856628109) -> (continuation of Deferred@1935081929 after retry RPC after error@1038110775) -> (continuation of Deferred@1439499308 after retry RPC after error@508931691) -> (continuation of Deferred@1439153441 after retry RPC after error@1780035142) -> (continuation of Deferred@1472768845 after retry RPC after error@1683845417) -> (continuation of Deferred@211547702 after retry RPC after error@794002173)))
07:05:53.014 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.30.122.61:45247, [1ms] received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [21ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [42ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [63ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [82ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [101ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [102ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [121ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [122ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [222ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [222ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, deferred=Deferred@142150689(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/2042931768@2bccb6b2 -> (continuation of Deferred@1286901281 after retry RPC after error@856628109) -> (continuation of Deferred@1935081929 after retry RPC after error@1038110775) -> (continuation of Deferred@1439499308 after retry RPC after error@508931691) -> (continuation of Deferred@1439153441 after retry RPC after error@1780035142) -> (continuation of Deferred@1472768845 after retry RPC after error@1683845417) -> (continuation of Deferred@211547702 after retry RPC after error@794002173), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1562323791@3dd3c699 -> (continuation of Deferred@1286901281 after retry RPC after error@856628109) -> (continuation of Deferred@1935081929 after retry RPC after error@1038110775) -> (continuation of Deferred@1439499308 after retry RPC after error@508931691) -> (continuation of Deferred@1439153441 after retry RPC after error@1780035142) -> (continuation of Deferred@1472768845 after retry RPC after error@1683845417) -> (continuation of Deferred@211547702 after retry RPC after error@794002173)))
07:05:53.030 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.032 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.050 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.052 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.071 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.090 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.092 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.110 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.112 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.150 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.152 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.190 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.190 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.192 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.256889 32314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:53.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.257100 32314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:53.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.257784 32314 leader_election.cc:288] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.258139 32291 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.258329 32291 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.258435 32252 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 5 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102" is_pre_election: true
07:05:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.258692 32252 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in term 4.
07:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.259034 32291 leader_election.cc:288] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 75aa2aca8a1d47b2b09e9bdd5474f9cb (127.30.122.62:42411), fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247)
07:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.259199 32084 leader_election.cc:302] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters:
07:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.259379 32082 leader_election.cc:334] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.259501 32314 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Leader pre-election won for term 5
07:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.259598 32314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.259703 32314 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 4 FOLLOWER]: Advancing to term 5
07:05:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.260028 32108 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a08014ec6614098a83031094f54a102" candidate_term: 5 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" is_pre_election: true
07:05:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.260808 32224 leader_election.cc:334] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.261898 32314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.262233 32108 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 5 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6a08014ec6614098a83031094f54a102 in current term 5: Already voted for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in this term.
07:05:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.262517 32314 leader_election.cc:288] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 5 election: Requested vote from peers fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247), 6a08014ec6614098a83031094f54a102 (127.30.122.60:37495)
07:05:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.262907 32252 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" candidate_term: 5 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "6a08014ec6614098a83031094f54a102"
07:05:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.263093 32252 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 4 FOLLOWER]: Advancing to term 5
07:05:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.263170 32224 leader_election.cc:302] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a08014ec6614098a83031094f54a102; no voters: 75aa2aca8a1d47b2b09e9bdd5474f9cb, fea5e5cdb708445faa58d8fde854b964
07:05:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.263844 32082 leader_election.cc:334] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111)
07:05:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.265220 32252 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 75aa2aca8a1d47b2b09e9bdd5474f9cb in term 5.
07:05:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.265404 32291 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 5 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
07:05:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.265581 32084 leader_election.cc:302] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a08014ec6614098a83031094f54a102, 75aa2aca8a1d47b2b09e9bdd5474f9cb; no voters: fea5e5cdb708445faa58d8fde854b964
07:05:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.265869 32314 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 5 FOLLOWER]: Leader election won for term 5
07:05:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.266239 32314 raft_consensus.cc:686] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [term 5 LEADER]: Becoming Leader. State: Replica: 75aa2aca8a1d47b2b09e9bdd5474f9cb, State: Running, Role: LEADER
07:05:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.266615 32314 consensus_queue.cc:237] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } }
07:05:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.268035 32319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 75aa2aca8a1d47b2b09e9bdd5474f9cb. Latest consensus state: current_term: 5 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.268325 32319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: LEADER
07:05:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.268942 32321 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:05:53.270 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.271626 32321 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236]
07:05:53.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.272493 32321 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0]
07:05:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.274569 32321 tablet_loader.cc:96] loaded metadata for tablet 4ff786b793784c1384710916b2184a07 (table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0])
07:05:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.274991 32321 tablet_loader.cc:96] loaded metadata for tablet 91583874abd345379149bc88532892ee (table kudu_system.kudu_transactions [id=7651e23a883d486dbb076c57f4074236])
07:05:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.275328 32321 tablet_loader.cc:96] loaded metadata for tablet bd855767784149e3b544eb1adaf714d0 (table txn_manager_fallback_rolling [id=7a7e12f7a46a40ee9863f028bde605c0])
07:05:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.275596 32321 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:05:53.276 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495; will retry
07:05:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.276535 32321 catalog_manager.cc:1259] Loaded cluster ID: fba1bfb12da94b5398d8c4f2d7866345
07:05:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.276635 32321 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:05:53.276 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=245), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [1ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [21ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [61ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [121ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [122ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [161ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [161ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [241ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [245ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [246ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, deferred=Deferred@697721298(state=PENDING, result=null, callback=(continuation of Deferred@1375808271 after retry RPC after error@1555329859) -> (continuation of Deferred@1398844723 after retry RPC after error@1231212156) -> (continuation of Deferred@316726540 after retry RPC after error@1831967352) -> (continuation of Deferred@997112792 after retry RPC after error@1688245223) -> (continuation of Deferred@1891379906 after retry RPC after error@289381623) -> (continuation of Deferred@1598117235 after retry RPC after error@1058986711) -> (continuation of Deferred@778378700 after retry RPC after error@1129374931), errback=(continuation of Deferred@1375808271 after retry RPC after error@1555329859) -> (continuation of Deferred@1398844723 after retry RPC after error@1231212156) -> (continuation of Deferred@316726540 after retry RPC after error@1831967352) -> (continuation of Deferred@997112792 after retry RPC after error@1688245223) -> (continuation of Deferred@1891379906 after retry RPC after error@289381623) -> (continuation of Deferred@1598117235 after retry RPC after error@1058986711) -> (continuation of Deferred@778378700 after retry RPC after error@1129374931)))
07:05:53.277 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=245), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [1ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [21ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [61ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [121ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [122ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [161ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [161ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [241ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.60:37495: OK, [245ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, [246ms] delaying RPC due to: Service unavailable: Master config (127.30.122.61:45247,127.30.122.62:42411,127.30.122.60:37495) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, deferred=Deferred@697721298(state=PENDING, result=null, callback=(continuation of Deferred@1375808271 after retry RPC after error@1555329859) -> (continuation of Deferred@1398844723 after retry RPC after error@1231212156) -> (continuation of Deferred@316726540 after retry RPC after error@1831967352) -> (continuation of Deferred@997112792 after retry RPC after error@1688245223) -> (continuation of Deferred@1891379906 after retry RPC after error@289381623) -> (continuation of Deferred@1598117235 after retry RPC after error@1058986711) -> (continuation of Deferred@778378700 after retry RPC after error@1129374931), errback=(continuation of Deferred@1375808271 after retry RPC after error@1555329859) -> (continuation of Deferred@1398844723 after retry RPC after error@1231212156) -> (continuation of Deferred@316726540 after retry RPC after error@1831967352) -> (continuation of Deferred@997112792 after retry RPC after error@1688245223) -> (continuation of Deferred@1891379906 after retry RPC after error@289381623) -> (continuation of Deferred@1598117235 after retry RPC after error@1058986711) -> (continuation of Deferred@778378700 after retry RPC after error@1129374931)))
07:05:53.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.278151 32321 catalog_manager.cc:1504] Loading token signing keys...
07:05:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.279227 32321 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb: Loaded TSK: 0
07:05:53.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.280341 32321 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:05:53.290 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.122.61:45247(127.30.122.61:45247)] connecting to peer
07:05:53.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.122.61:45247(127.30.122.61:45247)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.122.61:45247: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247
07:05:53.292 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:05:53.292 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.61:45247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.62:42411, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.122.60:37495, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.61:45247: Network error: Failed to connect to peer master-127.30.122.61:45247(127.30.122.61:45247): Connection refused: /127.30.122.61:45247, [2ms] Sub RPC ConnectToMaster: received response from server master-127.30.122.62:42411: OK, deferred=null) after lookup
07:05:53.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.317827 31542 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.026s from last keepalive heartbeat (effective timeout is 1.000s)
07:05:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.334681 31567 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.30.122.60:37495)
07:05:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.357937 32327 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
07:05:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.363999 32082 consensus_peers.cc:485] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb -> Peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Couldn't send request to peer fea5e5cdb708445faa58d8fde854b964. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.379937 32098 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41708:
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:05:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:05:53.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.384467 32252 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [term 5 FOLLOWER]: Refusing update from remote peer 75aa2aca8a1d47b2b09e9bdd5474f9cb: Log matching property violated. Preceding OpId in replica: term: 3 index: 13. Preceding OpId from leader: term: 5 index: 14. (index mismatch)
07:05:53.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.385136 32319 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
07:05:53.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.386632 32291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 75aa2aca8a1d47b2b09e9bdd5474f9cb. Latest consensus state: current_term: 5 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:53.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.386878 32291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.394228 32098 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:05:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.414021 32291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.414251 32291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a08014ec6614098a83031094f54a102 [sys.catalog]: This master's current role is: FOLLOWER
07:05:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.416769 32319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "75aa2aca8a1d47b2b09e9bdd5474f9cb" member_type: VOTER last_known_addr { host: "127.30.122.62" port: 42411 } } peers { permanent_uuid: "fea5e5cdb708445faa58d8fde854b964" member_type: VOTER last_known_addr { host: "127.30.122.61" port: 45247 } } peers { permanent_uuid: "6a08014ec6614098a83031094f54a102" member_type: VOTER last_known_addr { host: "127.30.122.60" port: 37495 } } }
07:05:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.417006 32319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb [sys.catalog]: This master's current role is: LEADER
07:05:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.448431 31803 heartbeater.cc:347] Connected to a master server at 127.30.122.60:37495
07:05:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.449769 31946 heartbeater.cc:347] Connected to a master server at 127.30.122.60:37495
07:05:53.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.452165 32242 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d19dc311d24345f39fd9b99f4b936983" instance_seqno: 1714115146156363) as {username='slave'} at 127.30.122.2:50565; Asking this server to re-register.
07:05:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.453330 31803 heartbeater.cc:464] Registering TS with master...
07:05:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.453672 32242 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8de35652c00d48feb4850f9c3cb5a27f" instance_seqno: 1714115146609654) as {username='slave'} at 127.30.122.3:44769; Asking this server to re-register.
07:05:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.454788 32242 ts_manager.cc:194] Registered new tserver with Master: d19dc311d24345f39fd9b99f4b936983 (127.30.122.2:42837)
07:05:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.455462 31946 heartbeater.cc:464] Registering TS with master...
07:05:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:53.457005 32242 ts_manager.cc:194] Registered new tserver with Master: 8de35652c00d48feb4850f9c3cb5a27f (127.30.122.3:32983)
07:05:53.465 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 608d7f45c8734fc880b32194a5ecf4f8 (127.30.122.1:40595): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1714115147 last_transition_timestamp: 1714115153
07:05:53.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:53.852133 32082 consensus_peers.cc:485] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb -> Peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Couldn't send request to peer fea5e5cdb708445faa58d8fde854b964. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
07:05:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:54.330246 32082 consensus_peers.cc:485] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb -> Peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Couldn't send request to peer fea5e5cdb708445faa58d8fde854b964. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
07:05:54.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.417912 31804 heartbeater.cc:503] Master 127.30.122.62:42411 was elected leader, sending a full tablet report...
07:05:54.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.439045 31947 heartbeater.cc:503] Master 127.30.122.62:42411 was elected leader, sending a full tablet report...
07:05:54.585 [DEBUG - main] (Connection.java:575) [peer master-127.30.122.61:45247(127.30.122.61:45247)] disconnecting while in state TERMINATED
07:05:54.585 [DEBUG - main] (Connection.java:575) [peer master-127.30.122.62:42411(127.30.122.62:42411)] disconnecting while in state READY
07:05:54.586 [DEBUG - main] (Connection.java:575) [peer 608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)] disconnecting while in state READY
07:05:54.586 [DEBUG - main] (Connection.java:575) [peer master-127.30.122.60:37495(127.30.122.60:37495)] disconnecting while in state READY
07:05:54.587 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)] handling channelInactive
07:05:54.587 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 608d7f45c8734fc880b32194a5ecf4f8(127.30.122.1:40595)] cleaning up while in state READY due to: connection closed
07:05:54.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:05:54.587 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.122.62:42411(127.30.122.62:42411)] handling channelInactive
07:05:54.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.62:42411(127.30.122.62:42411)] cleaning up while in state READY due to: connection closed
07:05:54.588 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.122.60:37495(127.30.122.60:37495)] handling channelInactive
07:05:54.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.122.60:37495(127.30.122.60:37495)] cleaning up while in state READY due to: connection closed
07:05:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.589587 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 31502
07:05:54.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.599985 31655 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:54.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.721079 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 31502
07:05:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.737784 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 31663
07:05:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.744268 31798 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.838236 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 31663
07:05:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.851856 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 31808
07:05:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:54.854769 32082 consensus_peers.cc:485] T 00000000000000000000000000000000 P 75aa2aca8a1d47b2b09e9bdd5474f9cb -> Peer fea5e5cdb708445faa58d8fde854b964 (127.30.122.61:45247): Couldn't send request to peer fea5e5cdb708445faa58d8fde854b964. Status: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
07:05:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.858755 31941 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.952205 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 31808
07:05:54.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.965400 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 32066
07:05:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.967883 32066 master_runner.cc:186] Error getting master registration for 127.30.122.61:45247: Network error: Client connection negotiation failed: client connection to 127.30.122.61:45247: connect: Connection refused (error 111),
07:05:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.968163 32066 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:54.971338 32138 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:55.082438 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 32066
07:05:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:05:55.098341 32225 connection.cc:535] server connection from 127.0.0.1:34138 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:55.098578 31208 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu pid 32209
07:05:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:55.104156 32282 generic_service.cc:196] Checking for leaks (request via RPC)
07:05:55.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:05:55.207907 31208 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIiZZ1l/build/asan/bin/kudu with pid 32209
07:05:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:05:55Z chronyd exiting
07:05:55.348 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)