16:19:09.258 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:19:09.258 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task81Iy5_/test-tmp
16:19:09.258 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
16:19:09.258 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task81Iy5_/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
16:19:09.259 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:19:09.338 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:19:09.339 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:19:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.339365 6851 mini_oidc.cc:152] Starting JWKS server
16:19:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.340090 6851 webserver.cc:460] Webserver started at http://127.0.0.1:37629/ using document root <none> and password file <none>
16:19:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.340168 6851 mini_oidc.cc:189] Starting OIDC Discovery server
16:19:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.340340 6851 webserver.cc:460] Webserver started at http://127.0.0.1:45101/ using document root <none> and password file <none>
16:19:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-06T16:19:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:19:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-06T16:19:09Z Disabled control of system clock
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.351291 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/logs
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/info.pb
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.254:45293
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.254
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
16:19:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
16:19:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.427155 6862 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.427325 6862 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.427357 6862 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428679 6862 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428722 6862 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
16:19:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428741 6862 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428761 6862 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428779 6862 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428797 6862 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.428812 6862 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37629/jwks.json
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.430280 6862 master_runner.cc:386] Master server non-default flags:
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.254:45293
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/info.pb
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.254
16:19:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6862
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/logs
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.430486 6862 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.430737 6862 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:09.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.435571 6868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.436211 6870 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.436892 6867 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.437618 6862 server_base.cc:1034] running on GCE node
16:19:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.437824 6862 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.438023 6862 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:09.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.439157 6862 hybrid_clock.cc:648] HybridClock initialized: now 1715012349439141 us; error 36 us; skew 500 ppm
16:19:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.440321 6862 webserver.cc:460] Webserver started at http://127.6.176.254:41183/ using document root <none> and password file <none>
16:19:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.440465 6862 fs_manager.cc:359] Metadata directory not provided
16:19:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.440500 6862 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.440572 6862 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.441442 6862 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/instance:
16:19:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e6c685e7c034f80b8ea330e23a0dd49"
16:19:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.441702 6862 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal/instance:
16:19:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e6c685e7c034f80b8ea330e23a0dd49"
16:19:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.442793 6862 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.002s sys 0.000s
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.443464 6876 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.443635 6862 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.443713 6862 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e6c685e7c034f80b8ea330e23a0dd49"
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.443763 6862 fs_report.cc:383] FS layout report
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/data
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.459811 6862 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.460067 6862 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.460157 6862 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.464113 6862 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.254:45293
16:19:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.464141 6938 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.254:45293 every 8 connection(s)
16:19:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.464411 6862 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/info.pb
16:19:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.465126 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 6862
16:19:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.465210 6851 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal/instance
16:19:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.465255 6939 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.465904 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/logs
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data/info.pb
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.253:37807
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.253
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
16:19:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.466785 6939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.472630 6939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.473013 6878 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.176.253:37807: connect: Connection refused (error 111)
16:19:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.473430 6939 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.176.253:37807: Network error: Client connection negotiation failed: client connection to 127.6.176.253:37807: connect: Connection refused (error 111)
16:19:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.522663 6939 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } attempt: 1
16:19:09.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.523415 6939 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.176.253:37807: Network error: Client connection negotiation failed: client connection to 127.6.176.253:37807: connect: Connection refused (error 111)
16:19:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.545841 6941 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.546006 6941 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.546038 6941 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547458 6941 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547506 6941 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547525 6941 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547541 6941 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547561 6941 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547577 6941 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.547591 6941 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37629/jwks.json
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.549598 6941 master_runner.cc:386] Master server non-default flags:
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal
16:19:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.253:37807
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data/info.pb
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.253
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6941
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/logs
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.549810 6941 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.550052 6941 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.554740 6952 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.555405 6950 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.556038 6949 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.556663 6941 server_base.cc:1034] running on GCE node
16:19:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.556885 6941 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.557130 6941 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.558261 6941 hybrid_clock.cc:648] HybridClock initialized: now 1715012349558241 us; error 34 us; skew 500 ppm
16:19:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.559450 6941 webserver.cc:460] Webserver started at http://127.6.176.253:36767/ using document root <none> and password file <none>
16:19:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.559671 6941 fs_manager.cc:359] Metadata directory not provided
16:19:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.559721 6941 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.559832 6941 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.560740 6941 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data/instance:
16:19:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.561107 6941 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal/instance:
16:19:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.562222 6941 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.000s sys 0.001s
16:19:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.562819 6958 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.562994 6941 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.563068 6941 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.563133 6941 fs_report.cc:383] FS layout report
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data/data
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.591591 6939 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } attempt: 2
16:19:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.592268 6939 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.176.253:37807: Network error: Client connection negotiation failed: client connection to 127.6.176.253:37807: connect: Connection refused (error 111)
16:19:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.592860 6941 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.593176 6941 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.593271 6941 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.597304 6941 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.253:37807
16:19:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.597329 7020 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.253:37807 every 8 connection(s)
16:19:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.597620 6941 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/data/info.pb
16:19:09.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.598354 7021 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:09.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.599622 7021 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.600607 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 6941
16:19:09.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.600687 6851 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-1/wal/instance
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.601015 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/logs
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data/info.pb
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.252:41507
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.252
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
16:19:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
16:19:09.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.604534 7021 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.607919 7021 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.608249 6961 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.608631 7021 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.176.252:41507: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.657830 7021 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } attempt: 1
16:19:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.658635 7021 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.176.252:41507: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.678673 7024 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.678833 7024 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.678866 7024 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680267 7024 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680308 7024 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680330 7024 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680349 7024 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680364 7024 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680380 7024 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.680395 7024 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37629/jwks.json
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.681866 7024 master_runner.cc:386] Master server non-default flags:
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.252:41507
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data/info.pb
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.252
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.7024
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/logs
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.682091 7024 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.682348 7024 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.683434 6939 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } attempt: 3
16:19:09.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.686612 6939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.686980 7031 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.687278 6939 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.176.252:41507: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.687716 7034 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.688338 7032 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.689002 7024 server_base.cc:1034] running on GCE node
16:19:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.689177 7024 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.689369 7024 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.690510 7024 hybrid_clock.cc:648] HybridClock initialized: now 1715012349690494 us; error 31 us; skew 500 ppm
16:19:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.691567 7024 webserver.cc:460] Webserver started at http://127.6.176.252:33253/ using document root <none> and password file <none>
16:19:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.691731 7024 fs_manager.cc:359] Metadata directory not provided
16:19:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.691766 7024 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.691840 7024 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.692634 7024 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data/instance:
16:19:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c69ff318c75a4f21bbe508dbc453e7d1"
16:19:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.692883 7024 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal/instance:
16:19:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c69ff318c75a4f21bbe508dbc453e7d1"
16:19:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.694007 7024 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.001s sys 0.000s
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.694588 7040 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.694747 7024 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.694830 7024 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c69ff318c75a4f21bbe508dbc453e7d1"
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.694877 7024 fs_report.cc:383] FS layout report
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal
16:19:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data/data
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.706054 7024 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.706272 7024 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.706352 7024 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.710255 7024 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.252:41507
16:19:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.710283 7102 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.252:41507 every 8 connection(s)
16:19:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.710548 7024 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/data/info.pb
16:19:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.711305 7103 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.712558 7103 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.715165 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 7024
16:19:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.715251 6851 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-2/wal/instance
16:19:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.717074 7103 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.718426 6939 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } attempt: 1
16:19:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.720047 7103 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } has no permanent_uuid. Determining permanent_uuid...
16:19:09.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.724347 7103 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: Bootstrap starting.
16:19:09.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.724571 6939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Bootstrap starting.
16:19:09.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.725291 7103 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: Neither blocks nor log segments found. Creating new log.
16:19:09.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.725582 7103 log.cc:824] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: Log is configured to *not* fsync() on all Append() calls
16:19:09.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.725736 6939 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Neither blocks nor log segments found. Creating new log.
16:19:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.725968 6939 log.cc:824] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Log is configured to *not* fsync() on all Append() calls
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.726253 7103 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: No bootstrap required, opened a new log
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.726564 6939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: No bootstrap required, opened a new log
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.726787 7021 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } attempt: 2
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727528 7103 raft_consensus.cc:348] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727617 7103 raft_consensus.cc:374] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727638 7103 raft_consensus.cc:729] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c69ff318c75a4f21bbe508dbc453e7d1, State: Initialized, Role: FOLLOWER
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727842 7103 consensus_queue.cc:260] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727845 6939 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727933 6939 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.727950 6939 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2e6c685e7c034f80b8ea330e23a0dd49, State: Initialized, Role: FOLLOWER
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.728018 6939 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.728124 7103 sys_catalog.cc:564] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: configured and running, proceeding with master startup.
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.728226 6939 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: configured and running, proceeding with master startup.
16:19:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.728207 7115 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.728271 7115 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.728116 7114 sys_catalog.cc:455] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.730051 7114 sys_catalog.cc:458] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.730120 7137 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.730157 7137 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.730554 7136 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.730698 7136 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.731288 6862 master_runner.cc:186] Error getting master registration for 127.6.176.253:37807: OK, instance_id { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" instance_seqno: 1715012349596320 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.733004 6862 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.734373 7021 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: Bootstrap starting.
16:19:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.735209 7021 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: Neither blocks nor log segments found. Creating new log.
16:19:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.735471 7021 log.cc:824] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: Log is configured to *not* fsync() on all Append() calls
16:19:09.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.736099 7021 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: No bootstrap required, opened a new log
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.735952 7024 master_runner.cc:186] Error getting master registration for 127.6.176.253:37807: OK, instance_id { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" instance_seqno: 1715012349596320 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738054 7024 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738083 7021 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738197 7021 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738225 7021 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5e7e6c4175254fcb8d2df83fcf28c501, State: Initialized, Role: FOLLOWER
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738339 7021 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738592 7141 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738636 7021 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: configured and running, proceeding with master startup.
16:19:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.738765 7141 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.740525 7152 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:09.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.740985 7152 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:09.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.742718 6941 master_runner.cc:263] No leader master found from master 5e7e6c4175254fcb8d2df83fcf28c501
16:19:09.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.742764 6941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.750928 7115 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751019 7115 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751251 7115 leader_election.cc:288] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751474 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" is_pre_election: true
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751616 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2e6c685e7c034f80b8ea330e23a0dd49 in term 0.
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751704 7066 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" is_pre_election: true
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751785 6878 leader_election.cc:302] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2e6c685e7c034f80b8ea330e23a0dd49, 5e7e6c4175254fcb8d2df83fcf28c501; no voters:
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751850 7066 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2e6c685e7c034f80b8ea330e23a0dd49 in term 0.
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751922 7115 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.751976 7115 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.752004 7115 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 0 FOLLOWER]: Advancing to term 1
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.753146 7115 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.753280 7115 leader_election.cc:288] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 1 election: Requested vote from peers 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.753448 7066 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c69ff318c75a4f21bbe508dbc453e7d1"
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.753508 7066 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 0 FOLLOWER]: Advancing to term 1
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.754263 7066 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2e6c685e7c034f80b8ea330e23a0dd49 in term 1.
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.754426 6879 leader_election.cc:302] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2e6c685e7c034f80b8ea330e23a0dd49, c69ff318c75a4f21bbe508dbc453e7d1; no voters:
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.754640 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.754703 6985 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 0 FOLLOWER]: Advancing to term 1
16:19:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.755434 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2e6c685e7c034f80b8ea330e23a0dd49 in term 1.
16:19:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.756939 7115 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 1 FOLLOWER]: Leader election won for term 1
16:19:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.757108 7115 raft_consensus.cc:686] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 1 LEADER]: Becoming Leader. State: Replica: 2e6c685e7c034f80b8ea330e23a0dd49, State: Running, Role: LEADER
16:19:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.757189 7115 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.757540 7115 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2e6c685e7c034f80b8ea330e23a0dd49. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.757627 7115 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: LEADER
16:19:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.760234 7156 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:19:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.760509 7156 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:19:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.762068 7066 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Refusing update from remote peer 2e6c685e7c034f80b8ea330e23a0dd49: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.762467 7154 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.763674 7114 sys_catalog.cc:455] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2e6c685e7c034f80b8ea330e23a0dd49. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.763751 7114 sys_catalog.cc:458] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.764989 6985 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Refusing update from remote peer 2e6c685e7c034f80b8ea330e23a0dd49: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.765343 7154 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.766429 7141 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2e6c685e7c034f80b8ea330e23a0dd49. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.766499 7141 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.767788 7154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.767861 7154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: LEADER
16:19:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.768374 7114 sys_catalog.cc:455] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.768450 7114 sys_catalog.cc:458] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.769148 7157 mvcc.cc:205] Tried to move back new op lower bound from 7024690584623091712 to 7024690584605974528. Current Snapshot: MvccSnapshot[applied={T|T < 7024690584623091712}]
16:19:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.769332 7158 mvcc.cc:205] Tried to move back new op lower bound from 7024690584623091712 to 7024690584605974528. Current Snapshot: MvccSnapshot[applied={T|T < 7024690584623091712}]
16:19:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.770519 7156 catalog_manager.cc:1347] Generated new cluster ID: 6c45e72d41634a21911c088926f215c8
16:19:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.770566 7156 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:19:09.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.781689 7154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.781795 7154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: LEADER
16:19:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.782470 7141 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.782554 7141 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.782779 7160 mvcc.cc:205] Tried to move back new op lower bound from 7024690584623091712 to 7024690584605974528. Current Snapshot: MvccSnapshot[applied={T|T < 7024690584623091712}]
16:19:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.783305 7156 catalog_manager.cc:1370] Generated new certificate authority record
16:19:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.784993 7156 catalog_manager.cc:1504] Loading token signing keys...
16:19:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.800928 7156 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Generated new TSK 0
16:19:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.801299 7156 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.812054 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/logs
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data/info.pb
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.193:0
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.176.193
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.193
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
16:19:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
16:19:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
16:19:09.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.928360 7168 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.928891 7168 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.929064 7168 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.931411 7168 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.931703 7168 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
16:19:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.931810 7168 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
16:19:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.931890 7168 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
16:19:09.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.931972 7168 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.176.193
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.934363 7168 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.193:0
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data/info.pb
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.193
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
16:19:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.7168
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.176.193
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/logs
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.935614 7168 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.936012 7168 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:09.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.941957 7174 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.942497 7173 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:09.943222 7176 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:09.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.943907 7168 server_base.cc:1034] running on GCE node
16:19:09.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.944082 7168 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:09.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.944308 7168 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:09.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.945439 7168 hybrid_clock.cc:648] HybridClock initialized: now 1715012349945419 us; error 34 us; skew 500 ppm
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.947588 7168 webserver.cc:460] Webserver started at http://127.6.176.193:33699/ using document root <none> and password file <none>
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.947782 7168 fs_manager.cc:359] Metadata directory not provided
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.947822 7168 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.947924 7168 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.949124 7168 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data/instance:
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89"
16:19:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.949842 7168 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal/instance:
16:19:09.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89"
16:19:09.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.951454 7168 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.001s sys 0.000s
16:19:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.952289 7182 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.952512 7168 fs_manager.cc:725] Time spent opening block manager: real 0.001s user 0.001s sys 0.000s
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.952589 7168 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89"
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.952643 7168 fs_report.cc:383] FS layout report
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data/data
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.978999 7168 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.979266 7168 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.979398 7168 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.980008 7168 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.980060 7168 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:19:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.980088 7168 ts_tablet_manager.cc:590] Registered 0 tablets
16:19:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.980104 7168 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:09.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.986685 7168 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.193:33387
16:19:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.987035 7168 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/data/info.pb
16:19:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.987596 7306 heartbeater.cc:347] Connected to a master server at 127.6.176.254:45293
16:19:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.987790 7306 heartbeater.cc:464] Registering TS with master...
16:19:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.988070 7306 heartbeater.cc:511] Master 127.6.176.254:45293 requested a full tablet report, sending...
16:19:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.988487 6893 ts_manager.cc:194] Registered new tserver with Master: db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387)
16:19:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.989455 6893 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.176.193:47845
16:19:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.990104 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 7168
16:19:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.990183 6851 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-0/wal/instance
16:19:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.990365 7303 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.193:33387 every 8 connection(s)
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.992004 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/logs
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data/info.pb
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.194:0
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.176.194
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.194
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994222 7307 heartbeater.cc:347] Connected to a master server at 127.6.176.253:37807
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994277 7307 heartbeater.cc:464] Registering TS with master...
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994335 7307 heartbeater.cc:511] Master 127.6.176.253:37807 requested a full tablet report, sending...
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994411 7305 heartbeater.cc:347] Connected to a master server at 127.6.176.252:41507
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994454 7305 heartbeater.cc:464] Registering TS with master...
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994490 7305 heartbeater.cc:511] Master 127.6.176.252:41507 requested a full tablet report, sending...
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994664 6975 ts_manager.cc:194] Registered new tserver with Master: db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387)
16:19:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:09.994802 7057 ts_manager.cc:194] Registered new tserver with Master: db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387)
16:19:10.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.069895 7310 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:10.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.070055 7310 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:10.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.070077 7310 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.071280 7310 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.071318 7310 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
16:19:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.071331 7310 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
16:19:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.071342 7310 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
16:19:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.071357 7310 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.176.194
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.072940 7310 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.194:0
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data/info.pb
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.194
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.7310
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.176.194
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/logs
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.073158 7310 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.073355 7310 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:10.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.077806 7315 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:10.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.078260 7318 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:10.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.079102 7310 server_base.cc:1034] running on GCE node
16:19:10.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.079602 7316 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:10.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.079867 7310 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:10.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.080119 7310 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:10.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.081262 7310 hybrid_clock.cc:648] HybridClock initialized: now 1715012350081246 us; error 31 us; skew 500 ppm
16:19:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.082494 7310 webserver.cc:460] Webserver started at http://127.6.176.194:34595/ using document root <none> and password file <none>
16:19:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.082698 7310 fs_manager.cc:359] Metadata directory not provided
16:19:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.082748 7310 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.082873 7310 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.083729 7310 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data/instance:
16:19:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "182418871c8945a58ec60b66c631ce72"
16:19:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:10 on dist-test-slave-69kz"
16:19:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.084196 7310 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal/instance:
16:19:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "182418871c8945a58ec60b66c631ce72"
16:19:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:10 on dist-test-slave-69kz"
16:19:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.085382 7310 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.000s sys 0.001s
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.086046 7324 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.086269 7310 fs_manager.cc:725] Time spent opening block manager: real 0.001s user 0.000s sys 0.001s
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.086345 7310 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "182418871c8945a58ec60b66c631ce72"
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:10 on dist-test-slave-69kz"
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.086410 7310 fs_report.cc:383] FS layout report
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal
16:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data/data
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.104825 7310 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.105098 7310 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.105217 7310 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.105706 7310 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.105741 7310 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.105775 7310 ts_tablet_manager.cc:590] Registered 0 tablets
16:19:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.105794 7310 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.111553 7310 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.194:38589
16:19:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.111752 7443 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.194:38589 every 8 connection(s)
16:19:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.112085 7310 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/data/info.pb
16:19:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.113487 7448 heartbeater.cc:347] Connected to a master server at 127.6.176.254:45293
16:19:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.113668 7448 heartbeater.cc:464] Registering TS with master...
16:19:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.113940 7448 heartbeater.cc:511] Master 127.6.176.254:45293 requested a full tablet report, sending...
16:19:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.114288 6892 ts_manager.cc:194] Registered new tserver with Master: 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.114823 6892 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.176.194:54427
16:19:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.116291 7447 heartbeater.cc:347] Connected to a master server at 127.6.176.252:41507
16:19:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.116426 7447 heartbeater.cc:464] Registering TS with master...
16:19:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.116559 7447 heartbeater.cc:511] Master 127.6.176.252:41507 requested a full tablet report, sending...
16:19:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.116875 7057 ts_manager.cc:194] Registered new tserver with Master: 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.117314 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 7310
16:19:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.117391 6851 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-1/wal/instance
16:19:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.118842 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/logs
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data/info.pb
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.195:0
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.176.195
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.195
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.118923 7449 heartbeater.cc:347] Connected to a master server at 127.6.176.253:37807
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.118973 7449 heartbeater.cc:464] Registering TS with master...
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.119007 7449 heartbeater.cc:511] Master 127.6.176.253:37807 requested a full tablet report, sending...
16:19:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.120342 6975 ts_manager.cc:194] Registered new tserver with Master: 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.197135 7452 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.197288 7452 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.197309 7452 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.198570 7452 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.198613 7452 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
16:19:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.198626 7452 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
16:19:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.198637 7452 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
16:19:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.198652 7452 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.176.195
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.199990 7452 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.195:0
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data/info.pb
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.195
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.7452
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.176.195
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/logs
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.200145 7452 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.200320 7452 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.205034 7460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.205698 7458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.206408 7452 server_base.cc:1034] running on GCE node
16:19:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.207013 7457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.207208 7452 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.207391 7452 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:10.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.208536 7452 hybrid_clock.cc:648] HybridClock initialized: now 1715012350208516 us; error 36 us; skew 500 ppm
16:19:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.209810 7452 webserver.cc:460] Webserver started at http://127.6.176.195:35145/ using document root <none> and password file <none>
16:19:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.209969 7452 fs_manager.cc:359] Metadata directory not provided
16:19:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.210006 7452 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.210081 7452 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.211007 7452 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data/instance:
16:19:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b43d33119284711a3bd190db61a3d41"
16:19:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:10 on dist-test-slave-69kz"
16:19:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.211266 7452 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal/instance:
16:19:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b43d33119284711a3bd190db61a3d41"
16:19:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:10 on dist-test-slave-69kz"
16:19:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.212384 7452 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.000s sys 0.002s
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.213013 7466 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.213160 7452 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.213224 7452 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b43d33119284711a3bd190db61a3d41"
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:10 on dist-test-slave-69kz"
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.213269 7452 fs_report.cc:383] FS layout report
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data/data
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235107 7452 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235313 7452 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235385 7452 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235845 7452 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235894 7452 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:19:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235924 7452 ts_tablet_manager.cc:590] Registered 0 tablets
16:19:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.235944 7452 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.242610 7452 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.195:42247
16:19:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.243144 7452 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/data/info.pb
16:19:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.243717 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 7452
16:19:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.243780 6851 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/ts-2/wal/instance
16:19:10.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.245311 7588 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.195:42247 every 8 connection(s)
16:19:10.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.246496 7590 heartbeater.cc:347] Connected to a master server at 127.6.176.254:45293
16:19:10.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.246682 7590 heartbeater.cc:464] Registering TS with master...
16:19:10.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.246942 7590 heartbeater.cc:511] Master 127.6.176.254:45293 requested a full tablet report, sending...
16:19:10.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.247778 6892 ts_manager.cc:194] Registered new tserver with Master: 6b43d33119284711a3bd190db61a3d41 (127.6.176.195:42247)
16:19:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.248307 6892 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.176.195:53231
16:19:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.249034 7589 heartbeater.cc:347] Connected to a master server at 127.6.176.252:41507
16:19:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.249087 7589 heartbeater.cc:464] Registering TS with master...
16:19:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.249145 7589 heartbeater.cc:511] Master 127.6.176.252:41507 requested a full tablet report, sending...
16:19:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.249419 7056 ts_manager.cc:194] Registered new tserver with Master: 6b43d33119284711a3bd190db61a3d41 (127.6.176.195:42247)
16:19:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.251163 7591 heartbeater.cc:347] Connected to a master server at 127.6.176.253:37807
16:19:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.251217 7591 heartbeater.cc:464] Registering TS with master...
16:19:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.251263 7591 heartbeater.cc:511] Master 127.6.176.253:37807 requested a full tablet report, sending...
16:19:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.251516 6975 ts_manager.cc:194] Registered new tserver with Master: 6b43d33119284711a3bd190db61a3d41 (127.6.176.195:42247)
16:19:10.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.258693 6851 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
16:19:10.259 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:19:10.259 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:19:10.260 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.176.254"
port: 45293
}
bound_http_address {
host: "127.6.176.254"
port: 41183
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.176.253"
port: 37807
}
bound_http_address {
host: "127.6.176.253"
port: 36767
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.176.252"
port: 41507
}
bound_http_address {
host: "127.6.176.252"
port: 33253
}
}
}
16:19:10.260 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:19:10.260 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.176.193"
port: 33387
}
bound_http_address {
host: "127.6.176.193"
port: 33699
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.176.194"
port: 38589
}
bound_http_address {
host: "127.6.176.194"
port: 34595
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.176.195"
port: 42247
}
bound_http_address {
host: "127.6.176.195"
port: 35145
}
}
}
16:19:10.260 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:19:10.262 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:19:10.263 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:10.265 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:10.266 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.176.253:37807(127.6.176.253:37807)] connecting to peer
16:19:10.265 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.176.254:45293(127.6.176.254:45293)] Successfully connected to peer
16:19:10.266 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.6.176.252:41507(127.6.176.252:41507)] Successfully connected to peer
16:19:10.267 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.6.176.254
16:19:10.268 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.6.176.252
16:19:10.272 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.176.253:37807(127.6.176.253:37807)] Successfully connected to peer
16:19:10.279 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.6.176.253
16:19:10.279 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507] using SASL/PLAIN
16:19:10.282 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x947379d3, L:/127.0.0.1:44146 - R:/127.6.176.254:45293] using SASL/PLAIN
16:19:10.289 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
16:19:10.289 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=26), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [17ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, deferred=null) after lookup
16:19:10.290 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc89079bc, L:/127.0.0.1:36566 - R:/127.6.176.253:37807] using SASL/PLAIN
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.290622 6892 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44146:
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
16:19:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
16:19:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.290946 6892 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
16:19:10.295 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8ee9dcafef7749c3b8b4f86036ae0508
16:19:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.300076 7516 tablet_service.cc:1467] Processing CreateTablet for tablet b85848acfd4945e5a49cbf3e13910daf (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
16:19:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.300072 7515 tablet_service.cc:1467] Processing CreateTablet for tablet 8a2569ac84124837b11cb7b59c463b26 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
16:19:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.300397 7515 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a2569ac84124837b11cb7b59c463b26. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.300484 7516 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b85848acfd4945e5a49cbf3e13910daf. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.302899 7602 tablet_bootstrap.cc:492] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: Bootstrap starting.
16:19:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.303398 7602 tablet_bootstrap.cc:654] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: Neither blocks nor log segments found. Creating new log.
16:19:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.303653 7602 log.cc:824] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: Log is configured to *not* fsync() on all Append() calls
16:19:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.304275 7602 tablet_bootstrap.cc:492] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: No bootstrap required, opened a new log
16:19:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.304335 7602 ts_tablet_manager.cc:1367] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: Time spent bootstrapping tablet: real 0.001s user 0.001s sys 0.000s
16:19:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.305631 7602 raft_consensus.cc:348] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.305744 7602 raft_consensus.cc:374] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.305773 7602 raft_consensus.cc:729] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b43d33119284711a3bd190db61a3d41, State: Initialized, Role: FOLLOWER
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.305866 7602 consensus_queue.cc:260] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.306118 7602 ts_tablet_manager.cc:1398] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.306159 7590 heartbeater.cc:503] Master 127.6.176.254:45293 was elected leader, sending a full tablet report...
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.306192 7602 tablet_bootstrap.cc:492] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41: Bootstrap starting.
16:19:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.306828 7602 tablet_bootstrap.cc:654] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41: Neither blocks nor log segments found. Creating new log.
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307498 7602 tablet_bootstrap.cc:492] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41: No bootstrap required, opened a new log
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307539 7602 ts_tablet_manager.cc:1367] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.001s
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307633 7602 raft_consensus.cc:348] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307682 7602 raft_consensus.cc:374] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307694 7602 raft_consensus.cc:729] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b43d33119284711a3bd190db61a3d41, State: Initialized, Role: FOLLOWER
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307723 7602 consensus_queue.cc:260] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.307823 7602 ts_tablet_manager.cc:1398] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
16:19:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.309790 7231 tablet_service.cc:1467] Processing CreateTablet for tablet 8a2569ac84124837b11cb7b59c463b26 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
16:19:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.309785 7232 tablet_service.cc:1467] Processing CreateTablet for tablet b85848acfd4945e5a49cbf3e13910daf (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
16:19:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.310158 7232 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b85848acfd4945e5a49cbf3e13910daf. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.310254 7231 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a2569ac84124837b11cb7b59c463b26. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.312880 7607 tablet_bootstrap.cc:492] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: Bootstrap starting.
16:19:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.313242 7374 tablet_service.cc:1467] Processing CreateTablet for tablet b85848acfd4945e5a49cbf3e13910daf (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
16:19:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.313427 7607 tablet_bootstrap.cc:654] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: Neither blocks nor log segments found. Creating new log.
16:19:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.313509 7374 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b85848acfd4945e5a49cbf3e13910daf. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.313733 7607 log.cc:824] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: Log is configured to *not* fsync() on all Append() calls
16:19:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.314458 7607 tablet_bootstrap.cc:492] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: No bootstrap required, opened a new log
16:19:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.314538 7607 ts_tablet_manager.cc:1367] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316267 7607 raft_consensus.cc:348] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316406 7607 raft_consensus.cc:374] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316431 7607 raft_consensus.cc:729] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db5b3b8e0ac84a2ab9af2ebc045e8e89, State: Initialized, Role: FOLLOWER
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316541 7607 consensus_queue.cc:260] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316706 7373 tablet_service.cc:1467] Processing CreateTablet for tablet 8a2569ac84124837b11cb7b59c463b26 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316776 7607 ts_tablet_manager.cc:1398] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: Time spent starting tablet: real 0.002s user 0.000s sys 0.003s
16:19:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316794 7373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a2569ac84124837b11cb7b59c463b26. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.316879 7306 heartbeater.cc:503] Master 127.6.176.254:45293 was elected leader, sending a full tablet report...
16:19:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.317262 7607 tablet_bootstrap.cc:492] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89: Bootstrap starting.
16:19:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.317706 7607 tablet_bootstrap.cc:654] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89: Neither blocks nor log segments found. Creating new log.
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318130 7609 tablet_bootstrap.cc:492] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72: Bootstrap starting.
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318331 7607 tablet_bootstrap.cc:492] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89: No bootstrap required, opened a new log
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318378 7607 ts_tablet_manager.cc:1367] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.001s
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318536 7609 tablet_bootstrap.cc:654] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72: Neither blocks nor log segments found. Creating new log.
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318665 7607 raft_consensus.cc:348] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318727 7607 raft_consensus.cc:374] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318750 7607 raft_consensus.cc:729] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db5b3b8e0ac84a2ab9af2ebc045e8e89, State: Initialized, Role: FOLLOWER
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318745 7609 log.cc:824] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72: Log is configured to *not* fsync() on all Append() calls
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318794 7607 consensus_queue.cc:260] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.318912 7607 ts_tablet_manager.cc:1398] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
16:19:10.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.319397 7609 tablet_bootstrap.cc:492] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72: No bootstrap required, opened a new log
16:19:10.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.319550 7609 ts_tablet_manager.cc:1367] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72: Time spent bootstrapping tablet: real 0.001s user 0.001s sys 0.000s
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.320838 7609 raft_consensus.cc:348] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.320979 7609 raft_consensus.cc:374] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.320999 7609 raft_consensus.cc:729] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 182418871c8945a58ec60b66c631ce72, State: Initialized, Role: FOLLOWER
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.321067 7609 consensus_queue.cc:260] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.321256 7609 ts_tablet_manager.cc:1398] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.321321 7448 heartbeater.cc:503] Master 127.6.176.254:45293 was elected leader, sending a full tablet report...
16:19:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.321347 7609 tablet_bootstrap.cc:492] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72: Bootstrap starting.
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.321983 7609 tablet_bootstrap.cc:654] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72: Neither blocks nor log segments found. Creating new log.
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322626 7609 tablet_bootstrap.cc:492] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72: No bootstrap required, opened a new log
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322667 7609 ts_tablet_manager.cc:1367] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72: Time spent bootstrapping tablet: real 0.001s user 0.001s sys 0.000s
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322765 7609 raft_consensus.cc:348] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322803 7609 raft_consensus.cc:374] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322816 7609 raft_consensus.cc:729] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 182418871c8945a58ec60b66c631ce72, State: Initialized, Role: FOLLOWER
16:19:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322845 7609 consensus_queue.cc:260] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.322964 7609 ts_tablet_manager.cc:1398] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
16:19:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.363262 7450 tablet.cc:2369] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.487912 7308 tablet.cc:2369] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.489022 7604 raft_consensus.cc:482] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.489106 7604 raft_consensus.cc:504] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.489377 7604 leader_election.cc:288] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387), 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.492730 7244 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a2569ac84124837b11cb7b59c463b26" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" is_pre_election: true
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.492733 7394 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a2569ac84124837b11cb7b59c463b26" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "182418871c8945a58ec60b66c631ce72" is_pre_election: true
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.492877 7244 raft_consensus.cc:2454] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 0.
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.492882 7394 raft_consensus.cc:2454] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 0.
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.493110 7469 leader_election.cc:302] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b43d33119284711a3bd190db61a3d41, db5b3b8e0ac84a2ab9af2ebc045e8e89; no voters:
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.493325 7604 raft_consensus.cc:2781] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.493378 7604 raft_consensus.cc:482] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.493402 7604 raft_consensus.cc:3037] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Advancing to term 1
16:19:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.494050 7604 raft_consensus.cc:504] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.494166 7604 leader_election.cc:288] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 election: Requested vote from peers db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387), 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.494347 7244 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a2569ac84124837b11cb7b59c463b26" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89"
16:19:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.494381 7394 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a2569ac84124837b11cb7b59c463b26" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "182418871c8945a58ec60b66c631ce72"
16:19:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.494426 7244 raft_consensus.cc:3037] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Advancing to term 1
16:19:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.494446 7394 raft_consensus.cc:3037] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Advancing to term 1
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.495174 7244 raft_consensus.cc:2454] T 8a2569ac84124837b11cb7b59c463b26 P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 1.
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.495174 7394 raft_consensus.cc:2454] T 8a2569ac84124837b11cb7b59c463b26 P 182418871c8945a58ec60b66c631ce72 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 1.
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.495325 7469 leader_election.cc:302] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b43d33119284711a3bd190db61a3d41, db5b3b8e0ac84a2ab9af2ebc045e8e89; no voters:
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.495411 7604 raft_consensus.cc:2781] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 1 FOLLOWER]: Leader election won for term 1
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.495549 7604 raft_consensus.cc:686] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [term 1 LEADER]: Becoming Leader. State: Replica: 6b43d33119284711a3bd190db61a3d41, State: Running, Role: LEADER
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.495621 7604 consensus_queue.cc:237] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.495828 7592 tablet.cc:2369] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.496389 6892 catalog_manager.cc:5484] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b43d33119284711a3bd190db61a3d41 (127.6.176.195). New cstate: current_term: 1 leader_uuid: "6b43d33119284711a3bd190db61a3d41" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } health_report { overall_health: HEALTHY } } }
16:19:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.724534 7615 raft_consensus.cc:482] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.724629 7615 raft_consensus.cc:504] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.724785 7615 leader_election.cc:288] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387), 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725005 7244 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b85848acfd4945e5a49cbf3e13910daf" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" is_pre_election: true
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725029 7394 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b85848acfd4945e5a49cbf3e13910daf" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "182418871c8945a58ec60b66c631ce72" is_pre_election: true
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725113 7394 raft_consensus.cc:2454] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 0.
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725113 7244 raft_consensus.cc:2454] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 0.
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725270 7469 leader_election.cc:302] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b43d33119284711a3bd190db61a3d41, db5b3b8e0ac84a2ab9af2ebc045e8e89; no voters:
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725350 7615 raft_consensus.cc:2781] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725406 7615 raft_consensus.cc:482] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.725431 7615 raft_consensus.cc:3037] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Advancing to term 1
16:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.726063 7615 raft_consensus.cc:504] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.726176 7615 leader_election.cc:288] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 election: Requested vote from peers db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387), 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.726341 7244 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b85848acfd4945e5a49cbf3e13910daf" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89"
16:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.726379 7394 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b85848acfd4945e5a49cbf3e13910daf" candidate_uuid: "6b43d33119284711a3bd190db61a3d41" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "182418871c8945a58ec60b66c631ce72"
16:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.726415 7244 raft_consensus.cc:3037] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Advancing to term 1
16:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.726445 7394 raft_consensus.cc:3037] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Advancing to term 1
16:19:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.727087 7394 raft_consensus.cc:2454] T b85848acfd4945e5a49cbf3e13910daf P 182418871c8945a58ec60b66c631ce72 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 1.
16:19:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.727087 7244 raft_consensus.cc:2454] T b85848acfd4945e5a49cbf3e13910daf P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b43d33119284711a3bd190db61a3d41 in term 1.
16:19:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.727259 7469 leader_election.cc:302] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b43d33119284711a3bd190db61a3d41, db5b3b8e0ac84a2ab9af2ebc045e8e89; no voters:
16:19:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.727341 7615 raft_consensus.cc:2781] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 1 FOLLOWER]: Leader election won for term 1
16:19:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.727397 7615 raft_consensus.cc:686] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [term 1 LEADER]: Becoming Leader. State: Replica: 6b43d33119284711a3bd190db61a3d41, State: Running, Role: LEADER
16:19:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.727439 7615 consensus_queue.cc:237] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.728111 6892 catalog_manager.cc:5484] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b43d33119284711a3bd190db61a3d41 (127.6.176.195). New cstate: current_term: 1 leader_uuid: "6b43d33119284711a3bd190db61a3d41" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } health_report { overall_health: HEALTHY } } }
16:19:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.731123 7136 catalog_manager.cc:1259] Loaded cluster ID: 6c45e72d41634a21911c088926f215c8
16:19:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.731174 7136 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: loading cluster ID for follower catalog manager: success
16:19:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.731995 7136 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: acquiring CA information for follower catalog manager: success
16:19:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.732246 7136 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.741478 7152 catalog_manager.cc:1259] Loaded cluster ID: 6c45e72d41634a21911c088926f215c8
16:19:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.741540 7152 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: loading cluster ID for follower catalog manager: success
16:19:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.742460 7152 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: acquiring CA information for follower catalog manager: success
16:19:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.742700 7152 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:10.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.929126 7615 consensus_queue.cc:1035] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [LEADER]: Connected to new peer: Peer: permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.931396 7604 consensus_queue.cc:1035] T 8a2569ac84124837b11cb7b59c463b26 P 6b43d33119284711a3bd190db61a3d41 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.951900 6892 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44156:
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
16:19:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
16:19:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:10.952075 6892 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.954690 7232 tablet_service.cc:1467] Processing CreateTablet for tablet e15f246491b64367af82d1e9ae969bdc (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.954766 7373 tablet_service.cc:1467] Processing CreateTablet for tablet e15f246491b64367af82d1e9ae969bdc (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.954865 7232 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e15f246491b64367af82d1e9ae969bdc. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.954919 7373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e15f246491b64367af82d1e9ae969bdc. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.954932 7516 tablet_service.cc:1467] Processing CreateTablet for tablet e15f246491b64367af82d1e9ae969bdc (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.955057 7516 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e15f246491b64367af82d1e9ae969bdc. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.955919 7637 tablet_bootstrap.cc:492] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: Bootstrap starting.
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.956383 7638 tablet_bootstrap.cc:492] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: Bootstrap starting.
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.956511 7639 tablet_bootstrap.cc:492] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: Bootstrap starting.
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.956718 7637 tablet_bootstrap.cc:654] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: Neither blocks nor log segments found. Creating new log.
16:19:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.956768 7638 tablet_bootstrap.cc:654] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: Neither blocks nor log segments found. Creating new log.
16:19:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.956949 7639 tablet_bootstrap.cc:654] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: Neither blocks nor log segments found. Creating new log.
16:19:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957413 7637 tablet_bootstrap.cc:492] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: No bootstrap required, opened a new log
16:19:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957489 7637 ts_tablet_manager.cc:1367] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
16:19:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957501 7638 tablet_bootstrap.cc:492] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: No bootstrap required, opened a new log
16:19:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957543 7638 ts_tablet_manager.cc:1367] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.001s
16:19:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957643 7637 raft_consensus.cc:348] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957679 7638 raft_consensus.cc:348] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957724 7637 raft_consensus.cc:374] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957734 7638 raft_consensus.cc:374] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957749 7637 raft_consensus.cc:729] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 182418871c8945a58ec60b66c631ce72, State: Initialized, Role: FOLLOWER
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957757 7638 raft_consensus.cc:729] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db5b3b8e0ac84a2ab9af2ebc045e8e89, State: Initialized, Role: FOLLOWER
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957798 7638 consensus_queue.cc:260] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957798 7637 consensus_queue.cc:260] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957890 7637 ts_tablet_manager.cc:1398] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957930 7638 ts_tablet_manager.cc:1398] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.957954 7619 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958034 7619 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958139 7623 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958204 7623 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958344 7639 tablet_bootstrap.cc:492] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: No bootstrap required, opened a new log
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958385 7639 ts_tablet_manager.cc:1367] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958495 7639 raft_consensus.cc:348] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958549 7639 raft_consensus.cc:374] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958570 7639 raft_consensus.cc:729] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b43d33119284711a3bd190db61a3d41, State: Initialized, Role: FOLLOWER
16:19:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958627 7639 consensus_queue.cc:260] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958802 7604 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.958866 7604 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:10.959333 7639 ts_tablet_manager.cc:1398] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
16:19:11.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.146828 7619 raft_consensus.cc:482] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:11.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.146921 7619 raft_consensus.cc:504] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:11.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.147246 7619 leader_election.cc:288] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387), 6b43d33119284711a3bd190db61a3d41 (127.6.176.195:42247)
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150332 7244 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e15f246491b64367af82d1e9ae969bdc" candidate_uuid: "182418871c8945a58ec60b66c631ce72" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" is_pre_election: true
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150426 7244 raft_consensus.cc:2454] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 182418871c8945a58ec60b66c631ce72 in term 0.
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150606 7327 leader_election.cc:302] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 182418871c8945a58ec60b66c631ce72, db5b3b8e0ac84a2ab9af2ebc045e8e89; no voters:
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150650 7534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e15f246491b64367af82d1e9ae969bdc" candidate_uuid: "182418871c8945a58ec60b66c631ce72" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b43d33119284711a3bd190db61a3d41" is_pre_election: true
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150728 7619 raft_consensus.cc:2781] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150772 7619 raft_consensus.cc:482] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150786 7619 raft_consensus.cc:3037] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 0 FOLLOWER]: Advancing to term 1
16:19:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.150771 7534 raft_consensus.cc:2454] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 182418871c8945a58ec60b66c631ce72 in term 0.
16:19:11.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.151367 7619 raft_consensus.cc:504] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:11.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.151487 7619 leader_election.cc:288] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [CANDIDATE]: Term 1 election: Requested vote from peers db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387), 6b43d33119284711a3bd190db61a3d41 (127.6.176.195:42247)
16:19:11.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.151648 7244 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e15f246491b64367af82d1e9ae969bdc" candidate_uuid: "182418871c8945a58ec60b66c631ce72" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89"
16:19:11.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.151664 7534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e15f246491b64367af82d1e9ae969bdc" candidate_uuid: "182418871c8945a58ec60b66c631ce72" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b43d33119284711a3bd190db61a3d41"
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.151729 7534 raft_consensus.cc:3037] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 0 FOLLOWER]: Advancing to term 1
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.151731 7244 raft_consensus.cc:3037] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 0 FOLLOWER]: Advancing to term 1
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.152410 7244 raft_consensus.cc:2454] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 182418871c8945a58ec60b66c631ce72 in term 1.
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.152410 7534 raft_consensus.cc:2454] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 182418871c8945a58ec60b66c631ce72 in term 1.
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.152609 7327 leader_election.cc:302] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 182418871c8945a58ec60b66c631ce72, db5b3b8e0ac84a2ab9af2ebc045e8e89; no voters:
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.152709 7619 raft_consensus.cc:2781] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 1 FOLLOWER]: Leader election won for term 1
16:19:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.152858 7619 raft_consensus.cc:686] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [term 1 LEADER]: Becoming Leader. State: Replica: 182418871c8945a58ec60b66c631ce72, State: Running, Role: LEADER
16:19:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.152966 7619 consensus_queue.cc:237] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } }
16:19:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.153282 7647 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: TxnStatusTablet state changed. Reason: New leader 182418871c8945a58ec60b66c631ce72. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.153441 7647 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: This TxnStatusTablet replica's current role is: LEADER
16:19:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.153671 6892 catalog_manager.cc:5484] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 reported cstate change: term changed from 0 to 1, leader changed from <none> to 182418871c8945a58ec60b66c631ce72 (127.6.176.194). New cstate: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } health_report { overall_health: UNKNOWN } } }
16:19:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.153792 7649 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
16:19:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.153884 7649 txn_status_manager.cc:929] Loading transaction status metadata into memory...
16:19:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.187371 7604 consensus_queue.cc:1035] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.188283 7245 raft_consensus.cc:1261] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89 [term 1 FOLLOWER]: Refusing update from remote peer 182418871c8945a58ec60b66c631ce72: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.188519 7647 consensus_queue.cc:1035] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.188892 7534 raft_consensus.cc:1261] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41 [term 1 FOLLOWER]: Refusing update from remote peer 182418871c8945a58ec60b66c631ce72: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.189157 7647 consensus_queue.cc:1035] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.189888 7623 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: TxnStatusTablet state changed. Reason: New leader 182418871c8945a58ec60b66c631ce72. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.189976 7623 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.190729 7622 mvcc.cc:205] Tried to move back new op lower bound from 7024690590464512000 to 7024690590323126272. Current Snapshot: MvccSnapshot[applied={T|T < 7024690590464512000}]
16:19:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.190760 7615 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: TxnStatusTablet state changed. Reason: New leader 182418871c8945a58ec60b66c631ce72. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.190834 7615 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.190833 7647 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.190894 7647 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: This TxnStatusTablet replica's current role is: LEADER
16:19:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.191723 7621 mvcc.cc:205] Tried to move back new op lower bound from 7024690590464512000 to 7024690590323126272. Current Snapshot: MvccSnapshot[applied={T|T < 7024690590464512000}]
16:19:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.192101 7604 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.192163 7604 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 6b43d33119284711a3bd190db61a3d41: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.192199 7604 consensus_queue.cc:1035] T b85848acfd4945e5a49cbf3e13910daf P 6b43d33119284711a3bd190db61a3d41 [LEADER]: Connected to new peer: Peer: permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.193315 7619 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.193395 7619 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P 182418871c8945a58ec60b66c631ce72: This TxnStatusTablet replica's current role is: LEADER
16:19:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.194835 7623 tablet_replica.cc:439] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "182418871c8945a58ec60b66c631ce72" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" member_type: VOTER last_known_addr { host: "127.6.176.193" port: 33387 } } peers { permanent_uuid: "182418871c8945a58ec60b66c631ce72" member_type: VOTER last_known_addr { host: "127.6.176.194" port: 38589 } } peers { permanent_uuid: "6b43d33119284711a3bd190db61a3d41" member_type: VOTER last_known_addr { host: "127.6.176.195" port: 42247 } } }
16:19:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.194926 7623 tablet_replica.cc:442] T e15f246491b64367af82d1e9ae969bdc P db5b3b8e0ac84a2ab9af2ebc045e8e89: This TxnStatusTablet replica's current role is: FOLLOWER
16:19:11.196 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
16:19:11.198 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8ee9dcafef7749c3b8b4f86036ae0508
16:19:11.198 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.176.254:45293
16:19:11.198 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:19:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.199589 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 6862
16:19:11.208 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.176.254:45293(127.6.176.254:45293)] handling channelInactive
16:19:11.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state READY due to: connection closed
16:19:11.209 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.346637 7590 heartbeater.cc:644] Failed to heartbeat to 127.6.176.254:45293 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.456212 7662 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2e6c685e7c034f80b8ea330e23a0dd49)
16:19:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.456329 7662 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.456498 7663 raft_consensus.cc:482] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2e6c685e7c034f80b8ea330e23a0dd49)
16:19:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.456561 7663 raft_consensus.cc:504] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.456728 7662 leader_election.cc:288] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.456784 7663 leader_election.cc:288] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807)
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457033 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" is_pre_election: true
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457111 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c69ff318c75a4f21bbe508dbc453e7d1 in term 1.
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457144 7066 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" is_pre_election: true
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457213 7066 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5e7e6c4175254fcb8d2df83fcf28c501 in term 1.
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.457304 7044 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457257 7042 leader_election.cc:302] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5e7e6c4175254fcb8d2df83fcf28c501, c69ff318c75a4f21bbe508dbc453e7d1; no voters:
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.457322 6962 leader_election.cc:334] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457439 7663 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Leader pre-election won for term 2
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457500 7663 raft_consensus.cc:482] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2e6c685e7c034f80b8ea330e23a0dd49)
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457523 7663 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 1 FOLLOWER]: Advancing to term 2
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457511 6961 leader_election.cc:302] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5e7e6c4175254fcb8d2df83fcf28c501, c69ff318c75a4f21bbe508dbc453e7d1; no voters: 2e6c685e7c034f80b8ea330e23a0dd49
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457621 7662 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Leader pre-election won for term 2
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457679 7662 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2e6c685e7c034f80b8ea330e23a0dd49)
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.457697 7662 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 1 FOLLOWER]: Advancing to term 2
16:19:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.457875 7044 leader_election.cc:334] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458367 7663 raft_consensus.cc:504] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458397 7662 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458496 7663 leader_election.cc:288] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 2 election: Requested vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807)
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458498 7662 leader_election.cc:288] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 2 election: Requested vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458636 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458721 6985 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c69ff318c75a4f21bbe508dbc453e7d1 in current term 2: Already voted for candidate 5e7e6c4175254fcb8d2df83fcf28c501 in this term.
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458779 7066 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c69ff318c75a4f21bbe508dbc453e7d1"
16:19:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.458849 7066 raft_consensus.cc:2379] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 5e7e6c4175254fcb8d2df83fcf28c501 in current term 2: Already voted for candidate c69ff318c75a4f21bbe508dbc453e7d1 in this term.
16:19:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.458997 6962 leader_election.cc:334] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.459044 6962 leader_election.cc:302] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5e7e6c4175254fcb8d2df83fcf28c501; no voters: 2e6c685e7c034f80b8ea330e23a0dd49, c69ff318c75a4f21bbe508dbc453e7d1
16:19:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.459110 7662 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
16:19:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.459280 7044 leader_election.cc:334] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.459319 7044 leader_election.cc:302] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c69ff318c75a4f21bbe508dbc453e7d1; no voters: 2e6c685e7c034f80b8ea330e23a0dd49, 5e7e6c4175254fcb8d2df83fcf28c501
16:19:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.459393 7663 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
16:19:11.704 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:11.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.705 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.6.176.254:45293(127.6.176.254:45293) for tablet Kudu Master: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.724 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:11.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.254:45293: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:11.744 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:11.745 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.745 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.745 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.254:45293: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.746 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:11.763 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.763924 7663 raft_consensus.cc:482] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.764025 7663 raft_consensus.cc:504] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.764168 7663 leader_election.cc:288] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807)
16:19:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.764438 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" is_pre_election: true
16:19:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.764544 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c69ff318c75a4f21bbe508dbc453e7d1 in term 2.
16:19:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.764717 7042 leader_election.cc:302] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5e7e6c4175254fcb8d2df83fcf28c501, c69ff318c75a4f21bbe508dbc453e7d1; no voters:
16:19:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.764974 7663 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Leader pre-election won for term 3
16:19:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.765029 7663 raft_consensus.cc:482] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.765050 7663 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 2 FOLLOWER]: Advancing to term 3
16:19:11.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.765 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.254:45293: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.765818 7044 leader_election.cc:334] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.765861 7663 raft_consensus.cc:504] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.765973 7663 leader_election.cc:288] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 3 election: Requested vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807)
16:19:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.766181 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:11.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.766273 6985 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 2 FOLLOWER]: Advancing to term 3
16:19:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.766530 7044 leader_election.cc:334] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.766930 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c69ff318c75a4f21bbe508dbc453e7d1 in term 3.
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767100 7042 leader_election.cc:302] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5e7e6c4175254fcb8d2df83fcf28c501, c69ff318c75a4f21bbe508dbc453e7d1; no voters: 2e6c685e7c034f80b8ea330e23a0dd49
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767190 7663 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 3 FOLLOWER]: Leader election won for term 3
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767314 7663 raft_consensus.cc:686] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [term 3 LEADER]: Becoming Leader. State: Replica: c69ff318c75a4f21bbe508dbc453e7d1, State: Running, Role: LEADER
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767406 7663 consensus_queue.cc:237] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767652 7668 sys_catalog.cc:455] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c69ff318c75a4f21bbe508dbc453e7d1. Latest consensus state: current_term: 3 leader_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767722 7668 sys_catalog.cc:458] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: This master's current role is: LEADER
16:19:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.767868 7669 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768162 7669 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768256 7669 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84]
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768429 7669 tablet_loader.cc:96] loaded metadata for tablet 8a2569ac84124837b11cb7b59c463b26 (table txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508])
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768489 7669 tablet_loader.cc:96] loaded metadata for tablet b85848acfd4945e5a49cbf3e13910daf (table txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508])
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768524 7669 tablet_loader.cc:96] loaded metadata for tablet e15f246491b64367af82d1e9ae969bdc (table kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84])
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768558 7669 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768623 7669 catalog_manager.cc:1259] Loaded cluster ID: 6c45e72d41634a21911c088926f215c8
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768666 7669 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:19:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768857 7669 catalog_manager.cc:1504] Loading token signing keys...
16:19:11.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.768961 7669 catalog_manager.cc:5868] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1: Loaded TSK: 0
16:19:11.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.769125 7669 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:19:11.783 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:11.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.254:45293: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293
16:19:11.786 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
16:19:11.787 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=83), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [2ms] received response from server master-127.6.176.254:45293: Network error: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: Network error: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: Network error: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: Network error: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: Network error: Failed to connect to peer master-127.6.176.254:45293(127.6.176.254:45293): Connection refused: /127.6.176.254:45293, [83ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: OK, deferred=Deferred@937624240(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@5c397b56 -> (continuation of Deferred@880660932 after retry RPC after error@722022015) -> (continuation of Deferred@80764716 after retry RPC after error@321156907) -> (continuation of Deferred@1225619322 after retry RPC after error@722846450), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@2dbac183 -> (continuation of Deferred@880660932 after retry RPC after error@722022015) -> (continuation of Deferred@80764716 after retry RPC after error@321156907) -> (continuation of Deferred@1225619322 after retry RPC after error@722846450))) after lookup
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.806818 7057 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33308:
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
16:19:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.807081 7057 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
16:19:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:11.877055 7044 consensus_peers.cc:485] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 -> Peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Couldn't send request to peer 2e6c685e7c034f80b8ea330e23a0dd49. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:19:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.878535 6985 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Refusing update from remote peer c69ff318c75a4f21bbe508dbc453e7d1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
16:19:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.878793 7668 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
16:19:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.879187 7662 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c69ff318c75a4f21bbe508dbc453e7d1. Latest consensus state: current_term: 3 leader_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.879273 7662 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.879703 7663 sys_catalog.cc:455] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.879796 7663 sys_catalog.cc:458] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 [sys.catalog]: This master's current role is: LEADER
16:19:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.879838 7662 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:11.879904 7662 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:12.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:12.195259 7448 heartbeater.cc:644] Failed to heartbeat to 127.6.176.254:45293 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:12.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:12.195701 7306 heartbeater.cc:644] Failed to heartbeat to 127.6.176.254:45293 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111)
16:19:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:12.396796 7044 consensus_peers.cc:485] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 -> Peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Couldn't send request to peer 2e6c685e7c034f80b8ea330e23a0dd49. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
16:19:12.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:12.889590 7044 consensus_peers.cc:485] T 00000000000000000000000000000000 P c69ff318c75a4f21bbe508dbc453e7d1 -> Peer 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293): Couldn't send request to peer 2e6c685e7c034f80b8ea330e23a0dd49. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.254:45293: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
16:19:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.195909 7305 heartbeater.cc:503] Master 127.6.176.252:41507 was elected leader, sending a full tablet report...
16:19:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.195919 7447 heartbeater.cc:503] Master 127.6.176.252:41507 was elected leader, sending a full tablet report...
16:19:13.212 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet b85848acfd4945e5a49cbf3e13910daf for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
16:19:13.213 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 8a2569ac84124837b11cb7b59c463b26 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
16:19:13.213 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=b85848acfd4945e5a49cbf3e13910daf@[182418871c8945a58ec60b66c631ce72(127.6.176.194:38589),6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)[L],db5b3b8e0ac84a2ab9af2ebc045e8e89(127.6.176.193:33387)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=8a2569ac84124837b11cb7b59c463b26@[182418871c8945a58ec60b66c631ce72(127.6.176.194:38589),6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)[L],db5b3b8e0ac84a2ab9af2ebc045e8e89(127.6.176.193:33387)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
16:19:13.213 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.176.252:41507, [1ms] Sub RPC GetTableLocations: received response from server master-127.6.176.252:41507: OK, deferred=null) after lookup
16:19:13.214 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)] connecting to peer
16:19:13.215 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)] Successfully connected to peer
16:19:13.225 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x526c32b5, L:/127.0.0.1:33390 - R:/127.6.176.195:42247] using TOKEN/null
16:19:13.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.227151 7567 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.6.176.254:45293)
16:19:13.237 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.176.254:45293
16:19:13.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.237725 6851 external_mini_cluster.cc:1326] Running /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task81Iy5_/build/release/bin/kudu
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/logs
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/info.pb
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.254:45293
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.254
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41183
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
16:19:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
16:19:13.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.320072 7710 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:13.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.320279 7710 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:13.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.320312 7710 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322271 7710 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322328 7710 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322351 7710 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322369 7710 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322392 7710 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322408 7710 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.322423 7710 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37629/jwks.json
16:19:13.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.324646 7710 master_runner.cc:386] Master server non-default flags:
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.176.212:38889
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.176.254:45293,127.6.176.253:37807,127.6.176.252:41507
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.176.254:45293
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37629/jwks.json
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/info.pb
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.176.254
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41183
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.7710
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/logs
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 15:45:53 UTC on efd1337a2aa3
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 162
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.324885 7710 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.325186 7710 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:19:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.330358 7715 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.330824 7716 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.331656 7718 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:13.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.332279 7710 server_base.cc:1034] running on GCE node
16:19:13.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.332547 7710 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:13.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.332830 7710 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:13.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.333978 7710 hybrid_clock.cc:648] HybridClock initialized: now 1715012353333981 us; error 51 us; skew 500 ppm
16:19:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.335361 7710 webserver.cc:460] Webserver started at http://127.6.176.254:41183/ using document root <none> and password file <none>
16:19:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.335568 7710 fs_manager.cc:359] Metadata directory not provided
16:19:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.335631 7710 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:13.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.336864 7710 fs_manager.cc:709] Time spent opening directory manager: real 0.001s user 0.001s sys 0.000s
16:19:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.337509 7724 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.337671 7710 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.337756 7710 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data,/tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e6c685e7c034f80b8ea330e23a0dd49"
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 16:19:09 on dist-test-slave-69kz"
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.338048 7710 fs_report.cc:383] FS layout report
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/wal
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/data
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.350162 7710 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.350450 7710 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.350569 7710 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:13.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.354671 7710 rpc_server.cc:307] RPC server started. Bound to: 127.6.176.254:45293
16:19:13.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.354701 7786 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.176.254:45293 every 8 connection(s)
16:19:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.355119 7710 server_base.cc:1166] Dumped server information to /tmp/dist-test-task81Iy5_/test-tmp/mini-kudu-cluster5194635287121360141/master-0/data/info.pb
16:19:13.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.356544 7788 sys_catalog.cc:263] Verifying existing consensus state
16:19:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.357283 7788 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Bootstrap starting.
16:19:13.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.362448 6851 external_mini_cluster.cc:1388] Started /tmp/dist-test-task81Iy5_/build/release/bin/kudu as pid 7710
16:19:13.362 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:13.362 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.176.252:41507
16:19:13.362 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:19:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.363107 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 7024
16:19:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.365908 7788 log.cc:824] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Log is configured to *not* fsync() on all Append() calls
16:19:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.368117 7788 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:19:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.368399 7788 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Bootstrap complete.
16:19:13.371 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:13.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.371582 7788 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:13.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.371829 7788 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2e6c685e7c034f80b8ea330e23a0dd49, State: Initialized, Role: FOLLOWER
16:19:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.371922 7788 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.372134 7791 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.372237 7791 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: FOLLOWER
16:19:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.372298 7751 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 1 FOLLOWER]: Advancing to term 3
16:19:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.373504 7791 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c69ff318c75a4f21bbe508dbc453e7d1. Latest consensus state: current_term: 3 leader_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.373596 7791 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: FOLLOWER
16:19:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.373657 7794 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.373696 7794 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: FOLLOWER
16:19:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.372179 7788 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: configured and running, proceeding with master startup.
16:19:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.373752 7728 connection.cc:441] server connection from 127.0.0.1:59814 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:59814 (request call id 0) could send its response
16:19:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.375571 7805 catalog_manager.cc:1259] Loaded cluster ID: 6c45e72d41634a21911c088926f215c8
16:19:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.375617 7805 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: loading cluster ID for follower catalog manager: success
16:19:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.376880 7805 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: acquiring CA information for follower catalog manager: success
16:19:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.377300 7805 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:13.371 [ERROR - kudu-nio-1] (Connection.java:452) [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:256) ~[netty-buffer-4.1.94.Final.jar:4.1.94.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:357) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:256)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:357)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
16:19:13.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state READY due to: [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507]
16:19:13.381 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.6.176.252:41507(127.6.176.252:41507) for tablet Kudu Master: [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507]
16:19:13.382 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.6.176.252:41507(127.6.176.252:41507)] handling channelInactive
16:19:13.384 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.254:45293(127.6.176.254:45293)] connecting to peer
16:19:13.385 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.385 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.6.176.254:45293(127.6.176.254:45293)] Successfully connected to peer
16:19:13.387 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.387 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.387 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.389432 7710 master_runner.cc:186] Error getting master registration for 127.6.176.252:41507: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111),
16:19:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.390874 7710 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:13.397 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x4bcd9281, L:/127.0.0.1:59838 - R:/127.6.176.254:45293] using TOKEN/null
16:19:13.398 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.404 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.406 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.424 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.444 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.446 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.464 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.504 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.505 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.524 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.526 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.527 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.527 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] sending RPC to server master-127.6.176.252:41507, [18ms] received response from server master-127.6.176.252:41507: Network error: [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507], [18ms] delaying RPC due to: Network error: [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507], [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [24ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [25ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [83ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [104ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [142ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [162ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [164ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, deferred=Deferred@1433471433(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@60970f88 -> (continuation of Deferred@1285899367 after retry RPC after error@190848161) -> (continuation of Deferred@1443001157 after retry RPC after error@238144834) -> (continuation of Deferred@1758541471 after retry RPC after error@1675602480) -> (continuation of Deferred@646083925 after retry RPC after error@643430170) -> (continuation of Deferred@1205684352 after retry RPC after error@1498276961) -> (continuation of Deferred@1817189056 after retry RPC after error@98095479), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@28ec0004 -> (continuation of Deferred@1285899367 after retry RPC after error@190848161) -> (continuation of Deferred@1443001157 after retry RPC after error@238144834) -> (continuation of Deferred@1758541471 after retry RPC after error@1675602480) -> (continuation of Deferred@646083925 after retry RPC after error@643430170) -> (continuation of Deferred@1205684352 after retry RPC after error@1498276961) -> (continuation of Deferred@1817189056 after retry RPC after error@98095479)))
16:19:13.528 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] sending RPC to server master-127.6.176.252:41507, [18ms] received response from server master-127.6.176.252:41507: Network error: [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507], [18ms] delaying RPC due to: Network error: [peer master-127.6.176.252:41507(127.6.176.252:41507)] unexpected exception from downstream on [id: 0xa2b65715, L:/127.0.0.1:33292 - R:/127.6.176.252:41507], [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [24ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [25ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [83ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [104ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [142ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [162ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [164ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, deferred=Deferred@1433471433(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@60970f88 -> (continuation of Deferred@1285899367 after retry RPC after error@190848161) -> (continuation of Deferred@1443001157 after retry RPC after error@238144834) -> (continuation of Deferred@1758541471 after retry RPC after error@1675602480) -> (continuation of Deferred@646083925 after retry RPC after error@643430170) -> (continuation of Deferred@1205684352 after retry RPC after error@1498276961) -> (continuation of Deferred@1817189056 after retry RPC after error@98095479), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@28ec0004 -> (continuation of Deferred@1285899367 after retry RPC after error@190848161) -> (continuation of Deferred@1443001157 after retry RPC after error@238144834) -> (continuation of Deferred@1758541471 after retry RPC after error@1675602480) -> (continuation of Deferred@646083925 after retry RPC after error@643430170) -> (continuation of Deferred@1205684352 after retry RPC after error@1498276961) -> (continuation of Deferred@1817189056 after retry RPC after error@98095479)))
16:19:13.544 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.546 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.564 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.566 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.582840 7589 heartbeater.cc:503] Master 127.6.176.252:41507 was elected leader, sending a full tablet report...
16:19:13.583 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.585 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.586019 7590 heartbeater.cc:347] Connected to a master server at 127.6.176.254:45293
16:19:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.586838 7741 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "6b43d33119284711a3bd190db61a3d41" instance_seqno: 1715012350240872) as {username='slave'} at 127.6.176.195:50671; Asking this server to re-register.
16:19:13.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.587100 7590 heartbeater.cc:464] Registering TS with master...
16:19:13.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.587392 7741 ts_manager.cc:194] Registered new tserver with Master: 6b43d33119284711a3bd190db61a3d41 (127.6.176.195:42247)
16:19:13.604 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.618604 7811 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader c69ff318c75a4f21bbe508dbc453e7d1)
16:19:13.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.618701 7811 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:13.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.618881 7811 leader_election.cc:288] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 2e6c685e7c034f80b8ea330e23a0dd49 (127.6.176.254:45293), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:13.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.619716 6961 leader_election.cc:334] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.621682 7751 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" is_pre_election: true
16:19:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.622004 6962 leader_election.cc:302] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5e7e6c4175254fcb8d2df83fcf28c501; no voters: 2e6c685e7c034f80b8ea330e23a0dd49, c69ff318c75a4f21bbe508dbc453e7d1
16:19:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.622107 7811 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
16:19:13.624 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.626 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.665 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.666 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807; will retry
16:19:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.673307 7794 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader c69ff318c75a4f21bbe508dbc453e7d1)
16:19:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.673424 7794 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.673696 7794 leader_election.cc:288] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.673934 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" is_pre_election: true
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.674029 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2e6c685e7c034f80b8ea330e23a0dd49 in term 3.
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.674024 7727 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.674232 7726 leader_election.cc:302] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2e6c685e7c034f80b8ea330e23a0dd49, 5e7e6c4175254fcb8d2df83fcf28c501; no voters:
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.674345 7794 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 3 FOLLOWER]: Leader pre-election won for term 4
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.674401 7794 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 3 FOLLOWER]: Starting leader election (detected failure of leader c69ff318c75a4f21bbe508dbc453e7d1)
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.674422 7794 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 3 FOLLOWER]: Advancing to term 4
16:19:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.674535 7727 leader_election.cc:334] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.675098 7794 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.675209 7794 leader_election.cc:288] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 4 election: Requested vote from peers 5e7e6c4175254fcb8d2df83fcf28c501 (127.6.176.253:37807), c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507)
16:19:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.675405 6985 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5e7e6c4175254fcb8d2df83fcf28c501"
16:19:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.675472 6985 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 3 FOLLOWER]: Advancing to term 4
16:19:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.675662 7727 leader_election.cc:334] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111)
16:19:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676101 6985 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2e6c685e7c034f80b8ea330e23a0dd49 in term 4.
16:19:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676316 7726 leader_election.cc:302] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2e6c685e7c034f80b8ea330e23a0dd49, 5e7e6c4175254fcb8d2df83fcf28c501; no voters: c69ff318c75a4f21bbe508dbc453e7d1
16:19:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676419 7794 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 4 FOLLOWER]: Leader election won for term 4
16:19:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676487 7794 raft_consensus.cc:686] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [term 4 LEADER]: Becoming Leader. State: Replica: 2e6c685e7c034f80b8ea330e23a0dd49, State: Running, Role: LEADER
16:19:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676568 7794 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } }
16:19:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676813 7791 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2e6c685e7c034f80b8ea330e23a0dd49. Latest consensus state: current_term: 4 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.676878 7791 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: LEADER
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677124 7816 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677385 7816 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508]
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677453 7816 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84]
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677578 7816 tablet_loader.cc:96] loaded metadata for tablet 8a2569ac84124837b11cb7b59c463b26 (table txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508])
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677618 7816 tablet_loader.cc:96] loaded metadata for tablet b85848acfd4945e5a49cbf3e13910daf (table txn_manager_fallback_rolling [id=8ee9dcafef7749c3b8b4f86036ae0508])
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677642 7816 tablet_loader.cc:96] loaded metadata for tablet e15f246491b64367af82d1e9ae969bdc (table kudu_system.kudu_transactions [id=9b452b95ab8a41b0b58cae855de42c84])
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677665 7816 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677703 7816 catalog_manager.cc:1259] Loaded cluster ID: 6c45e72d41634a21911c088926f215c8
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677717 7816 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677843 7816 catalog_manager.cc:1504] Loading token signing keys...
16:19:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677897 7816 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49: Loaded TSK: 0
16:19:13.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.677991 7816 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:19:13.684 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.6.176.252:41507(127.6.176.252:41507)] connecting to peer
16:19:13.685 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
16:19:13.685 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [2ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [21ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [40ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [41ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [121ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [140ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, deferred=Deferred@1103230886(state=PENDING, result=null, callback=(continuation of Deferred@738214247 after retry RPC after error@37854315) -> (continuation of Deferred@516937160 after retry RPC after error@1350397296) -> (continuation of Deferred@2119697761 after retry RPC after error@1886682117) -> (continuation of Deferred@276137558 after retry RPC after error@727220165) -> (continuation of Deferred@353699010 after retry RPC after error@1179829809) -> (continuation of Deferred@2057940891 after retry RPC after error@317149532), errback=(continuation of Deferred@738214247 after retry RPC after error@37854315) -> (continuation of Deferred@516937160 after retry RPC after error@1350397296) -> (continuation of Deferred@2119697761 after retry RPC after error@1886682117) -> (continuation of Deferred@276137558 after retry RPC after error@727220165) -> (continuation of Deferred@353699010 after retry RPC after error@1179829809) -> (continuation of Deferred@2057940891 after retry RPC after error@317149532))) after lookup
16:19:13.686 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.252:41507(127.6.176.252:41507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.176.252:41507: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.704483 7741 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59858:
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
16:19:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
16:19:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.705272 7788 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
16:19:13.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.759114 6985 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [term 4 FOLLOWER]: Refusing update from remote peer 2e6c685e7c034f80b8ea330e23a0dd49: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
16:19:13.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.759392 7791 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
16:19:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.759940 7811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2e6c685e7c034f80b8ea330e23a0dd49. Latest consensus state: current_term: 4 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.760049 7811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.760483 7794 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.760574 7794 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 [sys.catalog]: This master's current role is: LEADER
16:19:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.760648 7811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2e6c685e7c034f80b8ea330e23a0dd49" member_type: VOTER last_known_addr { host: "127.6.176.254" port: 45293 } } peers { permanent_uuid: "5e7e6c4175254fcb8d2df83fcf28c501" member_type: VOTER last_known_addr { host: "127.6.176.253" port: 37807 } } peers { permanent_uuid: "c69ff318c75a4f21bbe508dbc453e7d1" member_type: VOTER last_known_addr { host: "127.6.176.252" port: 41507 } } }
16:19:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.760708 7811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e7e6c4175254fcb8d2df83fcf28c501 [sys.catalog]: This master's current role is: FOLLOWER
16:19:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.761126 7727 consensus_peers.cc:485] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 -> Peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Couldn't send request to peer c69ff318c75a4f21bbe508dbc453e7d1. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:19:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.795570 7771 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:59838 (request call id 14) took 109 ms (client timeout 109 ms). Trace:
16:19:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.795672 7771 rpcz_store.cc:269] 0506 16:19:13.686383 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:19:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:13.686415 (+ 32us) service_pool.cc:225] Handling call
16:19:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:13.795552 (+109137us) inbound_call.cc:173] Queueing success response
16:19:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":303,"init.queue_time_us":27,"thread_start_us":219,"threads_started":4}
16:19:13.796 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [2ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [21ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [40ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [41ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [121ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [140ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [142ms] sending RPC to server master-127.6.176.254:45293, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1103230886(state=PENDING, result=null, callback=(continuation of Deferred@738214247 after retry RPC after error@37854315) -> (continuation of Deferred@516937160 after retry RPC after error@1350397296) -> (continuation of Deferred@2119697761 after retry RPC after error@1886682117) -> (continuation of Deferred@276137558 after retry RPC after error@727220165) -> (continuation of Deferred@353699010 after retry RPC after error@1179829809) -> (continuation of Deferred@2057940891 after retry RPC after error@317149532) -> (continuation of Deferred@795436246 after retry RPC@2136624196), errback=(continuation of Deferred@738214247 after retry RPC after error@37854315) -> (continuation of Deferred@516937160 after retry RPC after error@1350397296) -> (continuation of Deferred@2119697761 after retry RPC after error@1886682117) -> (continuation of Deferred@276137558 after retry RPC after error@727220165) -> (continuation of Deferred@353699010 after retry RPC after error@1179829809) -> (continuation of Deferred@2057940891 after retry RPC after error@317149532) -> (continuation of Deferred@795436246 after retry RPC@2136624196)))
16:19:13.797 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [2ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [21ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [40ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [41ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [121ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.6.176.254:45293,127.6.176.252:41507,127.6.176.253:37807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.254:45293, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.252:41507, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.176.253:37807, [140ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.254:45293: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.253:37807: OK, [142ms] sending RPC to server master-127.6.176.254:45293, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.176.252:41507: Network error: Failed to connect to peer master-127.6.176.252:41507(127.6.176.252:41507): Connection refused: /127.6.176.252:41507, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1103230886(state=PENDING, result=null, callback=(continuation of Deferred@738214247 after retry RPC after error@37854315) -> (continuation of Deferred@516937160 after retry RPC after error@1350397296) -> (continuation of Deferred@2119697761 after retry RPC after error@1886682117) -> (continuation of Deferred@276137558 after retry RPC after error@727220165) -> (continuation of Deferred@353699010 after retry RPC after error@1179829809) -> (continuation of Deferred@2057940891 after retry RPC after error@317149532) -> (continuation of Deferred@795436246 after retry RPC@2136624196), errback=(continuation of Deferred@738214247 after retry RPC after error@37854315) -> (continuation of Deferred@516937160 after retry RPC after error@1350397296) -> (continuation of Deferred@2119697761 after retry RPC after error@1886682117) -> (continuation of Deferred@276137558 after retry RPC after error@727220165) -> (continuation of Deferred@353699010 after retry RPC after error@1179829809) -> (continuation of Deferred@2057940891 after retry RPC after error@317149532) -> (continuation of Deferred@795436246 after retry RPC@2136624196)))
16:19:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:13.859532 7331 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.013s from last keepalive heartbeat (effective timeout is 1.000s)
16:19:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:13.864408 7829 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
16:19:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.054961 7771 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:59838 (request call id 15) took 250 ms (client timeout 250 ms). Trace:
16:19:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.055074 7771 rpcz_store.cc:269] 0506 16:19:13.804791 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:19:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:13.804849 (+ 58us) service_pool.cc:225] Handling call
16:19:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:14.054942 (+250093us) inbound_call.cc:173] Queueing success response
16:19:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:19:14.056 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1220075843(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@607d34b9, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@456b3b4))
16:19:14.056 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1220075843(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@607d34b9, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@456b3b4))
16:19:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.188373 7727 consensus_peers.cc:485] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 -> Peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Couldn't send request to peer c69ff318c75a4f21bbe508dbc453e7d1. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
16:19:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.199432 7448 heartbeater.cc:347] Connected to a master server at 127.6.176.254:45293
16:19:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.199419 7306 heartbeater.cc:347] Connected to a master server at 127.6.176.254:45293
16:19:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.199759 7740 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "182418871c8945a58ec60b66c631ce72" instance_seqno: 1715012350110082) as {username='slave'} at 127.6.176.194:60191; Asking this server to re-register.
16:19:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.199753 7741 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "db5b3b8e0ac84a2ab9af2ebc045e8e89" instance_seqno: 1715012349984815) as {username='slave'} at 127.6.176.193:54013; Asking this server to re-register.
16:19:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.200004 7448 heartbeater.cc:464] Registering TS with master...
16:19:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.200017 7306 heartbeater.cc:464] Registering TS with master...
16:19:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.200078 7306 heartbeater.cc:511] Master 127.6.176.254:45293 requested a full tablet report, sending...
16:19:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.200083 7448 heartbeater.cc:511] Master 127.6.176.254:45293 requested a full tablet report, sending...
16:19:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.200307 7741 ts_manager.cc:194] Registered new tserver with Master: db5b3b8e0ac84a2ab9af2ebc045e8e89 (127.6.176.193:33387)
16:19:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.200320 7740 ts_manager.cc:194] Registered new tserver with Master: 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589)
16:19:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.314960 7771 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:59838 (request call id 16) took 250 ms (client timeout 250 ms). Trace:
16:19:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.315060 7771 rpcz_store.cc:269] 0506 16:19:14.064819 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:19:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:14.064877 (+ 58us) service_pool.cc:225] Handling call
16:19:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:14.314944 (+250067us) inbound_call.cc:173] Queueing success response
16:19:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:19:14.315 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@700708630(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@26feb994, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@2f2f890e))
16:19:14.316 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@700708630(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@26feb994, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@2f2f890e))
16:19:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.574939 7771 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:59838 (request call id 17) took 250 ms (client timeout 250 ms). Trace:
16:19:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.575018 7771 rpcz_store.cc:269] 0506 16:19:14.324840 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:19:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:14.324921 (+ 81us) service_pool.cc:225] Handling call
16:19:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 16:19:14.574923 (+250002us) inbound_call.cc:173] Queueing success response
16:19:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:19:14.575 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@517293132(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@28986bc7, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@7bf95559))
16:19:14.576 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.6.176.254:45293, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@517293132(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/614755423@28986bc7, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/814848682@7bf95559))
16:19:14.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.688455 7727 consensus_peers.cc:485] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 -> Peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Couldn't send request to peer c69ff318c75a4f21bbe508dbc453e7d1. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.710435 7740 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59872:
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
16:19:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
16:19:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:14.710630 7740 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
16:19:14.715 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 182418871c8945a58ec60b66c631ce72 (127.6.176.194:38589): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1715012351 last_transition_timestamp: 1715012353
16:19:14.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.744540 6941 master_runner.cc:186] Error getting master registration for 127.6.176.252:41507: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111),
16:19:14.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:14.745776 6941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 16:19:15.164469 7727 consensus_peers.cc:485] T 00000000000000000000000000000000 P 2e6c685e7c034f80b8ea330e23a0dd49 -> Peer c69ff318c75a4f21bbe508dbc453e7d1 (127.6.176.252:41507): Couldn't send request to peer c69ff318c75a4f21bbe508dbc453e7d1. Status: Network error: Client connection negotiation failed: client connection to 127.6.176.252:41507: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
16:19:15.373 [DEBUG - main] (Connection.java:575) [peer master-127.6.176.252:41507(127.6.176.252:41507)] disconnecting while in state TERMINATED
16:19:15.374 [DEBUG - main] (Connection.java:575) [peer master-127.6.176.253:37807(127.6.176.253:37807)] disconnecting while in state READY
16:19:15.374 [DEBUG - main] (Connection.java:575) [peer master-127.6.176.254:45293(127.6.176.254:45293)] disconnecting while in state READY
16:19:15.374 [DEBUG - main] (Connection.java:575) [peer 6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)] disconnecting while in state READY
16:19:15.375 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.6.176.254:45293(127.6.176.254:45293)] handling channelInactive
16:19:15.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.176.254:45293(127.6.176.254:45293)] cleaning up while in state READY due to: connection closed
16:19:15.375 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:19:15.375 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.176.253:37807(127.6.176.253:37807)] handling channelInactive
16:19:15.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.176.253:37807(127.6.176.253:37807)] cleaning up while in state READY due to: connection closed
16:19:15.375 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)] handling channelInactive
16:19:15.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6b43d33119284711a3bd190db61a3d41(127.6.176.195:42247)] cleaning up while in state READY due to: connection closed
16:19:15.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:15.376438 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 7168
16:19:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:15.383863 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 7310
16:19:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:15.389389 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 7452
16:19:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:15.393654 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 7710
16:19:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 16:19:15.398770 6851 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task81Iy5_/build/release/bin/kudu with pid 6941
16:19:15.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-06T16:19:15Z chronyd exiting
16:19:15.447 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)