Unable to diagnose

02:23:06.368 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:23:06.369 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskVGqEZ7/test-tmp
02:23:06.369 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:23:06.369 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskVGqEZ7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:23:06.370 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692"
  extra_master_flags: "--txn_manager_enabled"
  extra_master_flags: "--raft_heartbeat_interval_ms=100"
  extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
  extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
  extra_tserver_flags: "--enable_txn_system_client_init=true"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

02:23:06.713 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
02:23:06.713 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

02:23:06.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:06.716639 20642 mini_oidc.cc:152] Starting JWKS server
02:23:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:06.720142 20642 webserver.cc:460] Webserver started at http://127.0.0.1:43623/ using document root <none> and password file <none>
02:23:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:06.720583 20642 mini_oidc.cc:189] Starting OIDC Discovery server
02:23:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:06.722451 20642 webserver.cc:460] Webserver started at http://127.0.0.1:42113/ using document root <none> and password file <none>
02:23:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:23:06Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:23:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:23:06Z Disabled control of system clock
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:06.790596 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/logs
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/info.pb
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.190:44353
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.190
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:23:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:23:07.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.110172 20654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:07.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.110805 20654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:07.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.111335 20654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:07.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.144196 20654 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:23:07.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.144670 20654 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:23:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.144858 20654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.145077 20654 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:23:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.145344 20654 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:23:07.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.145833 20654 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:23:07.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.146052 20654 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43623/jwks.json
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:07.183741 20654 master_runner.cc:386] Master server non-default flags:
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.190:44353
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/info.pb
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.190
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/logs
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:07.184973 20654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:07.186646 20654 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.202603 20660 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.202629 20661 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:07.207044 20654 server_base.cc:1034] running on GCE node
02:23:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:07.205505 20663 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.330924 20654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.333539 20654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:08.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.334899 20654 hybrid_clock.cc:648] HybridClock initialized: now 1714530188334857 us; error 60 us; skew 500 ppm
02:23:08.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.335729 20654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.347164 20654 webserver.cc:460] Webserver started at http://127.20.40.190:42245/ using document root <none> and password file <none>
02:23:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.348145 20654 fs_manager.cc:359] Metadata directory not provided
02:23:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.348403 20654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.348902 20654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:23:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.353922 20654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/instance:
02:23:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec33286a4c0b44fd90fcfc3e13d697cb"
02:23:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:08 on dist-test-slave-hmt2"
02:23:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.355047 20654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal/instance:
02:23:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec33286a4c0b44fd90fcfc3e13d697cb"
02:23:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:08 on dist-test-slave-hmt2"
02:23:08.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.363854 20654 fs_manager.cc:691] Time spent creating directory manager: real 0.008s	user 0.004s	sys 0.004s
02:23:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.371309 20670 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.372560 20654 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.372908 20654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec33286a4c0b44fd90fcfc3e13d697cb"
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:08 on dist-test-slave-hmt2"
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.373240 20654 fs_report.cc:383] FS layout report
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/data
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.443259 20654 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.444763 20654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.445197 20654 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.535567 20654 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.190:44353
02:23:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.535656 20731 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.190:44353 every 8 connection(s)
02:23:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.538344 20654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/info.pb
02:23:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.543325 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 20654
02:23:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.543712 20642 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal/instance
02:23:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.543661 20732 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.547163 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/logs
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data/info.pb
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.189:43575
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.189
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:23:08.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:23:08.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.558377 20732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } has no permanent_uuid. Determining permanent_uuid...
02:23:08.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.591820 20732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } has no permanent_uuid. Determining permanent_uuid...
02:23:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.595106 20674 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:08.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.599715 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.649256 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 1
02:23:08.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.654084 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.722488 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 2
02:23:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.727291 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:08.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.818778 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 3
02:23:08.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.823423 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:08.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.883500 20734 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:08.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.884200 20734 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:08.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.884954 20734 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.917568 20734 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:23:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.917973 20734 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:23:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.918257 20734 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.918538 20734 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:23:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.918808 20734 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:23:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.919075 20734 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:23:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.919288 20734 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43623/jwks.json
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.955999 20734 master_runner.cc:386] Master server non-default flags:
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.189:43575
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data/info.pb
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.189
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/logs
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.957574 20734 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.959345 20734 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.966899 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 4
02:23:08.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.971938 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:08.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.977430 20743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:09.271561 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 5
02:23:09.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:09.275414 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:09.823133 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 6
02:23:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:09.834304 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:10.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.977732 20744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:10.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:08.978559 20746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:10.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:08.978878 20734 server_base.cc:1034] running on GCE node
02:23:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.097422 20734 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.100078 20734 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.101435 20734 hybrid_clock.cc:648] HybridClock initialized: now 1714530190101392 us; error 61 us; skew 500 ppm
02:23:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.102306 20734 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.109162 20734 webserver.cc:460] Webserver started at http://127.20.40.189:38955/ using document root <none> and password file <none>
02:23:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.110211 20734 fs_manager.cc:359] Metadata directory not provided
02:23:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.110424 20734 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.110898 20734 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:23:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.116142 20734 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data/instance:
02:23:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e386f53f9dca4af0b5b66b91f47faa89"
02:23:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:10 on dist-test-slave-hmt2"
02:23:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.117264 20734 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal/instance:
02:23:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e386f53f9dca4af0b5b66b91f47faa89"
02:23:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:10 on dist-test-slave-hmt2"
02:23:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.124820 20734 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.002s
02:23:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.131086 20755 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.132568 20734 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.132923 20734 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e386f53f9dca4af0b5b66b91f47faa89"
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:10 on dist-test-slave-hmt2"
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.133265 20734 fs_report.cc:383] FS layout report
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data/data
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:10.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.186372 20734 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.187871 20734 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.188299 20734 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:10.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.272298 20734 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.189:43575
02:23:10.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.272375 20816 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.189:43575 every 8 connection(s)
02:23:10.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.274932 20734 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/data/info.pb
02:23:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.278386 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 20734
02:23:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.278888 20642 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-1/wal/instance
02:23:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.281010 20817 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.282159 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/logs
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/info.pb
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.188:36585
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.188
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:23:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:23:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.295368 20817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } has no permanent_uuid. Determining permanent_uuid...
02:23:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.316843 20817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } has no permanent_uuid. Determining permanent_uuid...
02:23:10.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.331341 20817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } has no permanent_uuid. Determining permanent_uuid...
02:23:10.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.333739 20759 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.338008 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.387459 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 1
02:23:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.391804 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.460186 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 2
02:23:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.464052 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.555533 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 3
02:23:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.559387 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.593518 20819 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.594151 20819 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.594645 20819 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.625932 20819 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:23:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.626231 20819 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:23:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.626387 20819 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.626592 20819 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:23:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.626785 20819 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:23:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.626974 20819 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:23:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.627120 20819 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43623/jwks.json
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.661810 20819 master_runner.cc:386] Master server non-default flags:
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.188:36585
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/info.pb
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.188
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/logs
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.663060 20819 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:10.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.664657 20819 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:10.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.683097 20829 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.702756 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 4
02:23:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.708503 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.894876 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } attempt: 7
02:23:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.910900 20732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } has no permanent_uuid. Determining permanent_uuid...
02:23:10.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.917984 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:10.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.976498 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 1
02:23:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.982028 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.007968 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 5
02:23:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:11.013674 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.063457 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 2
02:23:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:11.070130 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.155848 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 3
02:23:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:11.160950 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.301693 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 4
02:23:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:11.306689 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.561111 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 6
02:23:11.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:11.569197 20817 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.590395 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 5
02:23:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:11.595221 20732 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.40.188:36585: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:11.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.683600 20830 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:11.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:10.684015 20832 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:11.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:10.684870 20819 server_base.cc:1034] running on GCE node
02:23:11.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.819036 20819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:11.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.822116 20819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.823577 20819 hybrid_clock.cc:648] HybridClock initialized: now 1714530191823533 us; error 62 us; skew 500 ppm
02:23:11.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.824350 20819 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.830597 20819 webserver.cc:460] Webserver started at http://127.20.40.188:46763/ using document root <none> and password file <none>
02:23:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.831473 20819 fs_manager.cc:359] Metadata directory not provided
02:23:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.831656 20819 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.832053 20819 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:23:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.836939 20819 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/instance:
02:23:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf6ffc1cfe9b455ab3753f422a84e7db"
02:23:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:11 on dist-test-slave-hmt2"
02:23:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.838099 20819 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal/instance:
02:23:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf6ffc1cfe9b455ab3753f422a84e7db"
02:23:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:11 on dist-test-slave-hmt2"
02:23:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.845139 20819 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
02:23:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.850770 20844 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.851838 20819 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
02:23:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.852177 20819 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf6ffc1cfe9b455ab3753f422a84e7db"
02:23:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:11 on dist-test-slave-hmt2"
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.852546 20819 fs_report.cc:383] FS layout report
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/data
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.909698 20819 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.911307 20819 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.911873 20819 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.995559 20819 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.188:36585
02:23:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.995656 20905 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.188:36585 every 8 connection(s)
02:23:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:11.998258 20819 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/info.pb
02:23:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.000057 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 20819
02:23:12.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.000610 20642 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal/instance
02:23:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.004774 20906 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.025291 20906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } has no permanent_uuid. Determining permanent_uuid...
02:23:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.051133 20906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } has no permanent_uuid. Determining permanent_uuid...
02:23:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.061442 20906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } has no permanent_uuid. Determining permanent_uuid...
02:23:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.077580 20906 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Bootstrap starting.
02:23:12.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.083258 20906 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Neither blocks nor log segments found. Creating new log.
02:23:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.085417 20906 log.cc:824] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Log is configured to *not* fsync() on all Append() calls
02:23:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.089912 20906 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: No bootstrap required, opened a new log
02:23:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.111927 20906 raft_consensus.cc:348] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.112670 20906 raft_consensus.cc:374] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:12.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.112880 20906 raft_consensus.cc:729] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cf6ffc1cfe9b455ab3753f422a84e7db, State: Initialized, Role: FOLLOWER
02:23:12.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.113591 20906 consensus_queue.cc:260] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.116253 20915 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.117179 20915 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: FOLLOWER
02:23:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.118021 20906 sys_catalog.cc:564] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: configured and running, proceeding with master startup.
02:23:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.142858 20926 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:23:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.143245 20926 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:23:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.147888 20732 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 6
02:23:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.167347 20850 tablet.cc:2369] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:23:12.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.138327 20819 master_runner.cc:186] Error getting master registration for 127.20.40.190:44353: OK, instance_id { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" instance_seqno: 1714530188515930 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:23:12.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.168609 20819 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:23:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.173424 20732 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Bootstrap starting.
02:23:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.180666 20732 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Neither blocks nor log segments found. Creating new log.
02:23:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.183176 20732 log.cc:824] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Log is configured to *not* fsync() on all Append() calls
02:23:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.189757 20732 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: No bootstrap required, opened a new log
02:23:12.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.210052 20915 raft_consensus.cc:482] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:23:12.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.210655 20915 raft_consensus.cc:504] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.213508 20915 leader_election.cc:288] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575)
02:23:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.213665 20732 raft_consensus.cc:348] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.214617 20732 raft_consensus.cc:374] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:12.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.215078 20732 raft_consensus.cc:729] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ec33286a4c0b44fd90fcfc3e13d697cb, State: Initialized, Role: FOLLOWER
02:23:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.216105 20732 consensus_queue.cc:260] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.219897 20930 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:12.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.220965 20930 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.221835 20732 sys_catalog.cc:564] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: configured and running, proceeding with master startup.
02:23:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.222060 20697 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" is_pre_election: true
02:23:12.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.223006 20697 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cf6ffc1cfe9b455ab3753f422a84e7db in term 0.
02:23:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.224625 20847 leader_election.cc:302] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cf6ffc1cfe9b455ab3753f422a84e7db, ec33286a4c0b44fd90fcfc3e13d697cb; no voters: 
02:23:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.225917 20915 raft_consensus.cc:2781] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Leader pre-election won for term 1
02:23:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.226264 20915 raft_consensus.cc:482] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:23:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.226583 20915 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 0 FOLLOWER]: Advancing to term 1
02:23:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.235520 20915 raft_consensus.cc:504] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.241742 20697 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb"
02:23:12.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.242411 20697 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 0 FOLLOWER]: Advancing to term 1
02:23:12.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.242574 20915 leader_election.cc:288] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 1 election: Requested vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575)
02:23:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.250059 20697 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cf6ffc1cfe9b455ab3753f422a84e7db in term 1.
02:23:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.251116 20847 leader_election.cc:302] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cf6ffc1cfe9b455ab3753f422a84e7db, ec33286a4c0b44fd90fcfc3e13d697cb; no voters: 
02:23:12.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.252008 20915 raft_consensus.cc:2781] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 FOLLOWER]: Leader election won for term 1
02:23:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.254690 20915 raft_consensus.cc:686] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 LEADER]: Becoming Leader. State: Replica: cf6ffc1cfe9b455ab3753f422a84e7db, State: Running, Role: LEADER
02:23:12.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.255820 20915 consensus_queue.cc:237] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.286087 20936 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: New leader cf6ffc1cfe9b455ab3753f422a84e7db. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.288826 20936 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: LEADER
02:23:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.312007 20941 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:23:12.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.321496 20941 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:23:12.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.354861 20915 consensus_queue.cc:1035] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [LEADER]: Connected to new peer: Peer: permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:23:12.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.364130 20944 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:23:12.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.364856 20944 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:23:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.388903 20936 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:12.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.389770 20936 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: LEADER
02:23:12.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.350570 20654 master_runner.cc:186] Error getting master registration for 127.20.40.189:43575: OK, instance_id { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" instance_seqno: 1714530190253089 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:23:12.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.409505 20654 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:23:12.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.402978 20930 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader cf6ffc1cfe9b455ab3753f422a84e7db. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:12.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.415547 20930 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.425925 20949 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:12.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.427304 20949 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:12.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.435503 20941 catalog_manager.cc:1347] Generated new cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:12.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.435830 20941 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:23:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.476676 20941 catalog_manager.cc:1370] Generated new certificate authority record
02:23:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.479069 20941 catalog_manager.cc:1504] Loading token signing keys...
02:23:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.515228 20941 catalog_manager.cc:5857] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Generated new TSK 0
02:23:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.516376 20941 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.562052 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/logs
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data/info.pb
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.129:0
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.40.129
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.129
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:23:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:23:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.579532 20848 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.20.40.189:43575 timed out after 0.365s (SENT)
02:23:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.629578 20817 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } attempt: 7
02:23:12.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.727485 20817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: Bootstrap starting.
02:23:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.765609 20817 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: Neither blocks nor log segments found. Creating new log.
02:23:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.779058 20817 log.cc:824] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: Log is configured to *not* fsync() on all Append() calls
02:23:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:12.821997 20817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: No bootstrap required, opened a new log
02:23:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.951537 20848 leader_election.cc:334] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Timed out: RequestConsensusVote RPC to 127.20.40.189:43575 timed out after 0.353s (SENT)
02:23:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:12.962749 20848 leader_election.cc:334] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Timed out: RequestConsensusVote RPC to 127.20.40.189:43575 timed out after 0.365s (SENT)
02:23:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.015910 20817 raft_consensus.cc:348] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:13.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.023574 20817 raft_consensus.cc:374] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.025436 20817 raft_consensus.cc:729] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e386f53f9dca4af0b5b66b91f47faa89, State: Initialized, Role: FOLLOWER
02:23:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.030809 20817 consensus_queue.cc:260] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.051002 20956 raft_consensus.cc:482] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:23:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.051837 20956 raft_consensus.cc:504] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.057042 20697 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e386f53f9dca4af0b5b66b91f47faa89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" is_pre_election: true
02:23:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.059034 20871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e386f53f9dca4af0b5b66b91f47faa89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" is_pre_election: true
02:23:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.061185 20759 leader_election.cc:302] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e386f53f9dca4af0b5b66b91f47faa89; no voters: cf6ffc1cfe9b455ab3753f422a84e7db, ec33286a4c0b44fd90fcfc3e13d697cb
02:23:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.057201 20956 leader_election.cc:288] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585)
02:23:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.072284 20956 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.073259 20956 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: FOLLOWER
02:23:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.073808 20956 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:23:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.089118 20817 sys_catalog.cc:564] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: configured and running, proceeding with master startup.
02:23:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.092164 20780 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 0 FOLLOWER]: Advancing to term 1
02:23:13.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.092887 20778 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e386f53f9dca4af0b5b66b91f47faa89"
02:23:13.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.093622 20781 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e386f53f9dca4af0b5b66b91f47faa89"
02:23:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.102659 20781 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34264 (request call id 2) took 860 ms (client timeout 353 ms). Trace:
02:23:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.103160 20781 rpcz_store.cc:269] 0501 02:23:12.242322 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:12.242458 (+   136us) service_pool.cc:225] Handling call
02:23:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:13.102567 (+860109us) inbound_call.cc:173] Queueing success response
02:23:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:23:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.094271 20782 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e386f53f9dca4af0b5b66b91f47faa89" is_pre_election: true
02:23:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.105306 20782 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34264 (request call id 1) took 890 ms (client timeout 365 ms). Trace:
02:23:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.105791 20782 rpcz_store.cc:269] 0501 02:23:12.214656 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:12.214810 (+   154us) service_pool.cc:225] Handling call
02:23:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:13.105239 (+890429us) inbound_call.cc:173] Queueing success response
02:23:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:23:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.107230 20778 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34264 (request call id 5) took 509 ms (client timeout 353 ms). Trace:
02:23:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.107741 20778 rpcz_store.cc:269] 0501 02:23:12.598263 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:12.598409 (+   146us) service_pool.cc:225] Handling call
02:23:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:13.107160 (+508751us) inbound_call.cc:173] Queueing success response
02:23:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:23:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.094880 20779 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e386f53f9dca4af0b5b66b91f47faa89" is_pre_election: true
02:23:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.109083 20779 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34264 (request call id 4) took 512 ms (client timeout 365 ms). Trace:
02:23:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.109500 20779 rpcz_store.cc:269] 0501 02:23:12.597196 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:12.597375 (+   179us) service_pool.cc:225] Handling call
02:23:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:13.109020 (+511645us) inbound_call.cc:173] Queueing success response
02:23:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:23:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.112730 20759 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.20.40.188:36585, user_credentials={real_user=slave}} blocked reactor thread for 51631.8us
02:23:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.114651 20936 consensus_queue.cc:1035] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [LEADER]: Connected to new peer: Peer: permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:23:13.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.188395 20958 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cf6ffc1cfe9b455ab3753f422a84e7db. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.190107 20957 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.191092 20957 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: FOLLOWER
02:23:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.191716 20958 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: FOLLOWER
02:23:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.214988 20936 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.215811 20936 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: LEADER
02:23:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.219780 20952 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.220422 20952 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.221318 20952 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:13.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.240684 20981 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:23:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.280601 20952 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:13.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.281569 20952 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:23:13.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.281921 20952 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:23:13.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.282196 20952 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
02:23:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.282940 20952 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.40.129
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.348589 20952 tablet_server_runner.cc:78] Tablet server non-default flags:
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.129:0
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data/info.pb
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.129
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.40.129
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/logs
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.350230 20952 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.352403 20952 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.373271 20944 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.373824 20944 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: loading cluster ID for follower catalog manager: success
02:23:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.378552 20983 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.382102 20944 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: acquiring CA information for follower catalog manager: success
02:23:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:13.387861 20944 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:23:14.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.271113 20981 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.277992 20981 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: loading cluster ID for follower catalog manager: success
02:23:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.286159 20981 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: acquiring CA information for follower catalog manager: success
02:23:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.322742 20981 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:23:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:14.770300 20982 debug-util.cc:398] Leaking SignalData structure 0x7b0800014e80 after lost signal to thread 20952
02:23:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:14.830587 20952 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.455s	user 0.516s	sys 0.741s
02:23:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:14.837175 20952 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.462s	user 0.516s	sys 0.742s
02:23:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:13.379426 20984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:14.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:14.864114 20990 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.866881 20952 server_base.cc:1034] running on GCE node
02:23:14.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.868738 20952 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.877992 20952 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.879505 20952 hybrid_clock.cc:648] HybridClock initialized: now 1714530194879464 us; error 54 us; skew 500 ppm
02:23:14.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.880658 20952 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.909032 20952 webserver.cc:460] Webserver started at http://127.20.40.129:33715/ using document root <none> and password file <none>
02:23:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.910751 20952 fs_manager.cc:359] Metadata directory not provided
02:23:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.911121 20952 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.911832 20952 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:23:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.919498 20952 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data/instance:
02:23:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206"
02:23:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:14 on dist-test-slave-hmt2"
02:23:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.921129 20952 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal/instance:
02:23:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206"
02:23:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:14 on dist-test-slave-hmt2"
02:23:14.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.944099 20952 fs_manager.cc:691] Time spent creating directory manager: real 0.022s	user 0.009s	sys 0.008s
02:23:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.962980 20995 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.965626 20952 fs_manager.cc:725] Time spent opening block manager: real 0.017s	user 0.009s	sys 0.007s
02:23:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.966305 20952 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal
02:23:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206"
02:23:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:14 on dist-test-slave-hmt2"
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:14.966929 20952 fs_report.cc:383] FS layout report
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data/data
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.067461 20952 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.069809 20952 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.070676 20952 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.115278 20952 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:23:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.117028 20952 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.001s
02:23:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.117588 20952 ts_tablet_manager.cc:590] Registered 0 tablets
02:23:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.118003 20952 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
02:23:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.737272 20952 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.129:45293
02:23:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.737435 21120 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.129:45293 every 8 connection(s)
02:23:15.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.740940 20952 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/data/info.pb
02:23:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.750258 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 20952
02:23:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.750736 20642 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-0/wal/instance
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.763990 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/logs
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data/info.pb
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.130:0
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.40.130
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.130
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:23:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:23:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.802342 21121 heartbeater.cc:347] Connected to a master server at 127.20.40.188:36585
02:23:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.802811 21121 heartbeater.cc:464] Registering TS with master...
02:23:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.804124 21121 heartbeater.cc:511] Master 127.20.40.188:36585 requested a full tablet report, sending...
02:23:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.805119 21127 heartbeater.cc:347] Connected to a master server at 127.20.40.190:44353
02:23:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.805450 21127 heartbeater.cc:464] Registering TS with master...
02:23:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.807099 21127 heartbeater.cc:511] Master 127.20.40.190:44353 requested a full tablet report, sending...
02:23:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.807649 20861 ts_manager.cc:194] Registered new tserver with Master: 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293)
02:23:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.812007 20687 ts_manager.cc:194] Registered new tserver with Master: 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293)
02:23:15.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.814348 21122 heartbeater.cc:347] Connected to a master server at 127.20.40.189:43575
02:23:15.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.814730 21122 heartbeater.cc:464] Registering TS with master...
02:23:15.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.814747 20861 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.40.129:56291
02:23:15.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.815536 21122 heartbeater.cc:511] Master 127.20.40.189:43575 requested a full tablet report, sending...
02:23:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:15.818713 20772 ts_manager.cc:194] Registered new tserver with Master: 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293)
02:23:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.258261 21124 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:16.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.258982 21124 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:16.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.260030 21124 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.320751 21124 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:16.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.321791 21124 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:23:16.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.322141 21124 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:23:16.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.322749 21124 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
02:23:16.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.323509 21124 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.40.130
02:23:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:16.384997 21124 tablet_server_runner.cc:78] Tablet server non-default flags:
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.130:0
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data/info.pb
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.130
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.40.130
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/logs
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:16.386732 21124 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:16.389109 21124 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.411628 21134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:16.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:16.819137 21121 heartbeater.cc:503] Master 127.20.40.188:36585 was elected leader, sending a full tablet report...
02:23:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:17.810510 21133 debug-util.cc:398] Leaking SignalData structure 0x7b0800014e80 after lost signal to thread 21124
02:23:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.296288 21124 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.882s	user 0.681s	sys 1.042s
02:23:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.296700 21124 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.882s	user 0.681s	sys 1.042s
02:23:18.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:16.415303 21135 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:18.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.297926 21136 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1881 milliseconds
02:23:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.299029 21124 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:23:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.299073 21137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.302274 21124 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.304562 21124 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.305999 21124 hybrid_clock.cc:648] HybridClock initialized: now 1714530198305946 us; error 62 us; skew 500 ppm
02:23:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.306838 21124 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.313477 21124 webserver.cc:460] Webserver started at http://127.20.40.130:44309/ using document root <none> and password file <none>
02:23:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.314471 21124 fs_manager.cc:359] Metadata directory not provided
02:23:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.314754 21124 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:18.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.315258 21124 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:23:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.320307 21124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data/instance:
02:23:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3"
02:23:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:18 on dist-test-slave-hmt2"
02:23:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.321523 21124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal/instance:
02:23:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3"
02:23:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:18 on dist-test-slave-hmt2"
02:23:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.329660 21124 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.001s
02:23:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.335345 21148 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.336369 21124 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.336711 21124 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3"
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:18 on dist-test-slave-hmt2"
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.337056 21124 fs_report.cc:383] FS layout report
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data/data
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.395970 21124 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.397471 21124 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.398016 21124 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:18.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.408324 21124 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:23:18.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.408597 21124 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
02:23:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.408929 21124 ts_tablet_manager.cc:590] Registered 0 tablets
02:23:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.409104 21124 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
02:23:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.596740 21124 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.130:42347
02:23:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.597265 21272 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.130:42347 every 8 connection(s)
02:23:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.599310 21124 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/data/info.pb
02:23:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.600608 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 21124
02:23:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.601182 20642 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-1/wal/instance
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.612175 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/logs
02:23:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data/info.pb
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.131:0
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.40.131
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.131
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:23:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:23:18.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.624423 21273 heartbeater.cc:347] Connected to a master server at 127.20.40.188:36585
02:23:18.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.626528 21273 heartbeater.cc:464] Registering TS with master...
02:23:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.624655 21275 heartbeater.cc:347] Connected to a master server at 127.20.40.190:44353
02:23:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.627153 21275 heartbeater.cc:464] Registering TS with master...
02:23:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.627885 21273 heartbeater.cc:511] Master 127.20.40.188:36585 requested a full tablet report, sending...
02:23:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.628396 21275 heartbeater.cc:511] Master 127.20.40.190:44353 requested a full tablet report, sending...
02:23:18.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.631083 20861 ts_manager.cc:194] Registered new tserver with Master: a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.632216 21274 heartbeater.cc:347] Connected to a master server at 127.20.40.189:43575
02:23:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.632598 21274 heartbeater.cc:464] Registering TS with master...
02:23:18.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.632776 20861 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.40.130:39667
02:23:18.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.633546 21274 heartbeater.cc:511] Master 127.20.40.189:43575 requested a full tablet report, sending...
02:23:18.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.636183 20687 ts_manager.cc:194] Registered new tserver with Master: a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:18.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:18.637993 20772 ts_manager.cc:194] Registered new tserver with Master: a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:18.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.965214 21278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:18.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.965734 21278 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:18.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:18.966629 21278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.000375 21278 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.000996 21278 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:23:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.001180 21278 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:23:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.001335 21278 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
02:23:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.001906 21278 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.40.131
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:19.039256 21278 tablet_server_runner.cc:78] Tablet server non-default flags:
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.131:0
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data/info.pb
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.131
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.40.131
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/logs
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:19.040558 21278 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:19.042264 21278 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.061257 21288 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:19.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:19.636678 21273 heartbeater.cc:503] Master 127.20.40.188:36585 was elected leader, sending a full tablet report...
02:23:20.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.061689 21286 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:19.063149 21278 server_base.cc:1034] running on GCE node
02:23:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:19.062752 21285 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:20.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.261405 21278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:20.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.264662 21278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:20.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.266515 21278 hybrid_clock.cc:648] HybridClock initialized: now 1714530200266489 us; error 75 us; skew 500 ppm
02:23:20.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.267570 21278 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:20.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.281757 21278 webserver.cc:460] Webserver started at http://127.20.40.131:40641/ using document root <none> and password file <none>
02:23:20.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.283142 21278 fs_manager.cc:359] Metadata directory not provided
02:23:20.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.283449 21278 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.284063 21278 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:23:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.290735 21278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data/instance:
02:23:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de9196e9aa724b00b90c36fd643070bd"
02:23:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:20 on dist-test-slave-hmt2"
02:23:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.292418 21278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal/instance:
02:23:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de9196e9aa724b00b90c36fd643070bd"
02:23:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:20 on dist-test-slave-hmt2"
02:23:20.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.301556 21278 fs_manager.cc:691] Time spent creating directory manager: real 0.008s	user 0.010s	sys 0.000s
02:23:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.308123 21296 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:20.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.309592 21278 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
02:23:20.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.310071 21278 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal
02:23:20.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de9196e9aa724b00b90c36fd643070bd"
02:23:20.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:20 on dist-test-slave-hmt2"
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.310537 21278 fs_report.cc:383] FS layout report
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data/data
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:20.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.372817 21278 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.374673 21278 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.375252 21278 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.386474 21278 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:23:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.386922 21278 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
02:23:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.387358 21278 ts_tablet_manager.cc:590] Registered 0 tablets
02:23:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.387601 21278 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
02:23:20.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.624724 21278 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.131:44377
02:23:20.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.625200 21421 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.131:44377 every 8 connection(s)
02:23:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.627707 21278 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/data/info.pb
02:23:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.633338 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 21278
02:23:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.633911 20642 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/ts-2/wal/instance
02:23:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.656036 21422 heartbeater.cc:347] Connected to a master server at 127.20.40.188:36585
02:23:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.656531 21422 heartbeater.cc:464] Registering TS with master...
02:23:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.658453 21424 heartbeater.cc:347] Connected to a master server at 127.20.40.190:44353
02:23:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.658633 21422 heartbeater.cc:511] Master 127.20.40.188:36585 requested a full tablet report, sending...
02:23:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.658831 21424 heartbeater.cc:464] Registering TS with master...
02:23:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.659762 21424 heartbeater.cc:511] Master 127.20.40.190:44353 requested a full tablet report, sending...
02:23:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.663388 20687 ts_manager.cc:194] Registered new tserver with Master: de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377)
02:23:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.665303 20861 ts_manager.cc:194] Registered new tserver with Master: de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377)
02:23:20.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.666990 20861 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.40.131:53437
02:23:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.668720 21423 heartbeater.cc:347] Connected to a master server at 127.20.40.189:43575
02:23:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.669214 21423 heartbeater.cc:464] Registering TS with master...
02:23:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.670432 21423 heartbeater.cc:511] Master 127.20.40.189:43575 requested a full tablet report, sending...
02:23:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.673390 20772 ts_manager.cc:194] Registered new tserver with Master: de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377)
02:23:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.687954 20642 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:23:20.699 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
02:23:20.699 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

02:23:20.701 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.40.190"
      port: 44353
    }
    bound_http_address {
      host: "127.20.40.190"
      port: 42245
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.40.189"
      port: 43575
    }
    bound_http_address {
      host: "127.20.40.189"
      port: 38955
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.40.188"
      port: 36585
    }
    bound_http_address {
      host: "127.20.40.188"
      port: 46763
    }
  }
}

02:23:20.701 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

02:23:20.702 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.40.129"
      port: 45293
    }
    bound_http_address {
      host: "127.20.40.129"
      port: 33715
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.40.130"
      port: 42347
    }
    bound_http_address {
      host: "127.20.40.130"
      port: 44309
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.40.131"
      port: 44377
    }
    bound_http_address {
      host: "127.20.40.131"
      port: 40641
    }
  }
}

02:23:20.702 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:23:20.704 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
02:23:20.706 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:20.707 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:20.708 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.40.190:44353(127.20.40.190:44353)] Successfully connected to peer
02:23:20.711 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.40.190
02:23:20.727 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa119c794, L:/127.0.0.1:49054 - R:/127.20.40.190:44353] using SASL/PLAIN
02:23:20.733 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.40.188:36585(127.20.40.188:36585)] connecting to peer
02:23:20.733 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.40.189:43575(127.20.40.189:43575)] Successfully connected to peer
02:23:20.735 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.40.188:36585(127.20.40.188:36585)] Successfully connected to peer
02:23:20.737 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.40.189
02:23:20.737 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.40.188
02:23:20.755 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x921944e0, L:/127.0.0.1:43696 - R:/127.20.40.189:43575] using SASL/PLAIN
02:23:20.757 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x337a8b32, L:/127.0.0.1:37850 - R:/127.20.40.188:36585] using SASL/PLAIN
02:23:20.761 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:23:20.762 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=56), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [26ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [54ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [55ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, deferred=null) after lookup
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.765646 20861 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37850:
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   hash_schema {
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     num_buckets: 2
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     seed: 0
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:20.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:20.771981 20861 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:23:20.828 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table fa66dc7beb084580987b289db640f9aa
02:23:20.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.902781 21055 tablet_service.cc:1467] Processing CreateTablet for tablet f6becb12d0fa43b39fece0887b13d671 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:23:20.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.903788 21054 tablet_service.cc:1467] Processing CreateTablet for tablet 677c29065eaa44c49d687c21e5e9e194 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:23:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.905653 21055 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6becb12d0fa43b39fece0887b13d671. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.906925 21054 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 677c29065eaa44c49d687c21e5e9e194. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.906011 21355 tablet_service.cc:1467] Processing CreateTablet for tablet 677c29065eaa44c49d687c21e5e9e194 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:23:20.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.908262 21355 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 677c29065eaa44c49d687c21e5e9e194. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:20.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.909759 21356 tablet_service.cc:1467] Processing CreateTablet for tablet f6becb12d0fa43b39fece0887b13d671 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:23:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.911129 21356 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6becb12d0fa43b39fece0887b13d671. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:20.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.915663 21207 tablet_service.cc:1467] Processing CreateTablet for tablet 677c29065eaa44c49d687c21e5e9e194 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:23:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.919579 21208 tablet_service.cc:1467] Processing CreateTablet for tablet f6becb12d0fa43b39fece0887b13d671 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:23:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.921492 21207 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 677c29065eaa44c49d687c21e5e9e194. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.923700 21208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6becb12d0fa43b39fece0887b13d671. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.947292 21448 tablet_bootstrap.cc:492] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Bootstrap starting.
02:23:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.958875 21449 tablet_bootstrap.cc:492] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3: Bootstrap starting.
02:23:20.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.967240 21449 tablet_bootstrap.cc:654] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3: Neither blocks nor log segments found. Creating new log.
02:23:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.967662 21450 tablet_bootstrap.cc:492] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: Bootstrap starting.
02:23:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.968076 21448 tablet_bootstrap.cc:654] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Neither blocks nor log segments found. Creating new log.
02:23:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.970223 21448 log.cc:824] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Log is configured to *not* fsync() on all Append() calls
02:23:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.970494 21449 log.cc:824] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3: Log is configured to *not* fsync() on all Append() calls
02:23:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.975952 21450 tablet_bootstrap.cc:654] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: Neither blocks nor log segments found. Creating new log.
02:23:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.976722 21449 tablet_bootstrap.cc:492] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3: No bootstrap required, opened a new log
02:23:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.977051 21448 tablet_bootstrap.cc:492] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: No bootstrap required, opened a new log
02:23:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.977190 21449 ts_tablet_manager.cc:1367] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3: Time spent bootstrapping tablet: real 0.019s	user 0.012s	sys 0.003s
02:23:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.977614 21448 ts_tablet_manager.cc:1367] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Time spent bootstrapping tablet: real 0.031s	user 0.014s	sys 0.000s
02:23:20.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.979130 21450 log.cc:824] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: Log is configured to *not* fsync() on all Append() calls
02:23:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.985203 21450 tablet_bootstrap.cc:492] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: No bootstrap required, opened a new log
02:23:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:20.985821 21450 ts_tablet_manager.cc:1367] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: Time spent bootstrapping tablet: real 0.019s	user 0.009s	sys 0.005s
02:23:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.008023 21449 raft_consensus.cc:348] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.009114 21449 raft_consensus.cc:374] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.009402 21449 raft_consensus.cc:729] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a4c37e2ef7c34a4db2b8c5114b6083c3, State: Initialized, Role: FOLLOWER
02:23:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.008862 21448 raft_consensus.cc:348] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.010110 21448 raft_consensus.cc:374] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.010465 21448 raft_consensus.cc:729] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fd6af1f6d2445dfbf7c0c7df22ac206, State: Initialized, Role: FOLLOWER
02:23:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.010608 21449 consensus_queue.cc:260] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.011678 21448 consensus_queue.cc:260] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.025864 21450 raft_consensus.cc:348] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.028046 21450 raft_consensus.cc:374] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.028484 21450 raft_consensus.cc:729] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: de9196e9aa724b00b90c36fd643070bd, State: Initialized, Role: FOLLOWER
02:23:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.029500 21449 ts_tablet_manager.cc:1398] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3: Time spent starting tablet: real 0.052s	user 0.039s	sys 0.012s
02:23:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.029929 21450 consensus_queue.cc:260] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.032318 21448 ts_tablet_manager.cc:1398] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Time spent starting tablet: real 0.054s	user 0.037s	sys 0.012s
02:23:21.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.034799 21449 tablet_bootstrap.cc:492] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3: Bootstrap starting.
02:23:21.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.040367 21450 ts_tablet_manager.cc:1398] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: Time spent starting tablet: real 0.054s	user 0.039s	sys 0.008s
02:23:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.041345 21422 heartbeater.cc:503] Master 127.20.40.188:36585 was elected leader, sending a full tablet report...
02:23:21.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.046041 21450 tablet_bootstrap.cc:492] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd: Bootstrap starting.
02:23:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.049980 21448 tablet_bootstrap.cc:492] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Bootstrap starting.
02:23:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.054966 21450 tablet_bootstrap.cc:654] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd: Neither blocks nor log segments found. Creating new log.
02:23:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.056504 21448 tablet_bootstrap.cc:654] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Neither blocks nor log segments found. Creating new log.
02:23:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.073753 21448 tablet_bootstrap.cc:492] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206: No bootstrap required, opened a new log
02:23:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.074268 21448 ts_tablet_manager.cc:1367] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Time spent bootstrapping tablet: real 0.025s	user 0.015s	sys 0.000s
02:23:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.083729 21449 tablet_bootstrap.cc:654] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3: Neither blocks nor log segments found. Creating new log.
02:23:21.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.084952 21448 raft_consensus.cc:348] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.085826 21448 raft_consensus.cc:374] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.086239 21448 raft_consensus.cc:729] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fd6af1f6d2445dfbf7c0c7df22ac206, State: Initialized, Role: FOLLOWER
02:23:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.087009 21448 consensus_queue.cc:260] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.090387 21450 tablet_bootstrap.cc:492] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd: No bootstrap required, opened a new log
02:23:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.090535 21449 tablet_bootstrap.cc:492] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3: No bootstrap required, opened a new log
02:23:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.090893 21450 ts_tablet_manager.cc:1367] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd: Time spent bootstrapping tablet: real 0.045s	user 0.029s	sys 0.008s
02:23:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.091248 21449 ts_tablet_manager.cc:1367] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3: Time spent bootstrapping tablet: real 0.057s	user 0.012s	sys 0.001s
02:23:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.091612 21448 ts_tablet_manager.cc:1398] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Time spent starting tablet: real 0.010s	user 0.003s	sys 0.004s
02:23:21.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:21.092672 21128 tablet.cc:2369] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:23:21.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.093709 21450 raft_consensus.cc:348] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.094604 21450 raft_consensus.cc:374] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.094902 21450 raft_consensus.cc:729] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: de9196e9aa724b00b90c36fd643070bd, State: Initialized, Role: FOLLOWER
02:23:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.095474 21449 raft_consensus.cc:348] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.096248 21449 raft_consensus.cc:374] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.095836 21450 consensus_queue.cc:260] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.103025 21449 raft_consensus.cc:729] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a4c37e2ef7c34a4db2b8c5114b6083c3, State: Initialized, Role: FOLLOWER
02:23:21.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.104441 21449 consensus_queue.cc:260] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.108767 21449 ts_tablet_manager.cc:1398] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3: Time spent starting tablet: real 0.017s	user 0.005s	sys 0.000s
02:23:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.110664 21450 ts_tablet_manager.cc:1398] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd: Time spent starting tablet: real 0.019s	user 0.002s	sys 0.004s
02:23:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:21.127964 21276 tablet.cc:2369] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:23:21.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:21.164027 21426 tablet.cc:2369] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:23:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.223459 21455 raft_consensus.cc:482] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:23:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.223979 21455 raft_consensus.cc:504] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.226696 21455 leader_election.cc:288] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377), a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.239090 21454 raft_consensus.cc:482] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:23:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.239777 21454 raft_consensus.cc:504] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.240708 21376 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6becb12d0fa43b39fece0887b13d671" candidate_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de9196e9aa724b00b90c36fd643070bd" is_pre_election: true
02:23:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.241397 21376 raft_consensus.cc:2454] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fd6af1f6d2445dfbf7c0c7df22ac206 in term 0.
02:23:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.241484 21228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6becb12d0fa43b39fece0887b13d671" candidate_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" is_pre_election: true
02:23:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.242352 21228 raft_consensus.cc:2454] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fd6af1f6d2445dfbf7c0c7df22ac206 in term 0.
02:23:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.242890 20998 leader_election.cc:302] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9fd6af1f6d2445dfbf7c0c7df22ac206, de9196e9aa724b00b90c36fd643070bd; no voters: 
02:23:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.244449 21454 leader_election.cc:288] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293), de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377)
02:23:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.244956 21455 raft_consensus.cc:2781] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:23:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.245364 21455 raft_consensus.cc:482] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:23:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.245723 21455 raft_consensus.cc:3037] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Advancing to term 1
02:23:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.253918 21455 raft_consensus.cc:504] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.257274 21455 leader_election.cc:288] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [CANDIDATE]: Term 1 election: Requested vote from peers de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377), a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:21.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.259135 21376 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6becb12d0fa43b39fece0887b13d671" candidate_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de9196e9aa724b00b90c36fd643070bd"
02:23:21.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.259878 21376 raft_consensus.cc:3037] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Advancing to term 1
02:23:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.260506 21228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6becb12d0fa43b39fece0887b13d671" candidate_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3"
02:23:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.261179 21228 raft_consensus.cc:3037] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Advancing to term 1
02:23:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.263181 21375 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6becb12d0fa43b39fece0887b13d671" candidate_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de9196e9aa724b00b90c36fd643070bd" is_pre_election: true
02:23:21.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.267429 21376 raft_consensus.cc:2454] T f6becb12d0fa43b39fece0887b13d671 P de9196e9aa724b00b90c36fd643070bd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fd6af1f6d2445dfbf7c0c7df22ac206 in term 1.
02:23:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.268606 21228 raft_consensus.cc:2454] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fd6af1f6d2445dfbf7c0c7df22ac206 in term 1.
02:23:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.268821 20998 leader_election.cc:302] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9fd6af1f6d2445dfbf7c0c7df22ac206, de9196e9aa724b00b90c36fd643070bd; no voters: 
02:23:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.270073 21455 raft_consensus.cc:2781] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 1 FOLLOWER]: Leader election won for term 1
02:23:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.271404 21076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6becb12d0fa43b39fece0887b13d671" candidate_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" is_pre_election: true
02:23:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.274492 21455 raft_consensus.cc:686] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 1 LEADER]: Becoming Leader. State: Replica: 9fd6af1f6d2445dfbf7c0c7df22ac206, State: Running, Role: LEADER
02:23:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.275877 21455 consensus_queue.cc:237] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } }
02:23:21.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.293114 20861 catalog_manager.cc:5484] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129). New cstate: current_term: 1 leader_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } health_report { overall_health: UNKNOWN } } }
02:23:21.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.294715 21149 leader_election.cc:302] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a4c37e2ef7c34a4db2b8c5114b6083c3; no voters: 9fd6af1f6d2445dfbf7c0c7df22ac206, de9196e9aa724b00b90c36fd643070bd
02:23:21.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.295764 21454 raft_consensus.cc:2726] T f6becb12d0fa43b39fece0887b13d671 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:23:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.301674 21456 raft_consensus.cc:482] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:23:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.302394 21456 raft_consensus.cc:504] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.315251 21456 leader_election.cc:288] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293), a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.327652 21076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "677c29065eaa44c49d687c21e5e9e194" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" is_pre_election: true
02:23:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.328399 21076 raft_consensus.cc:2454] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 0.
02:23:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.330201 21228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "677c29065eaa44c49d687c21e5e9e194" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" is_pre_election: true
02:23:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.330376 21297 leader_election.cc:302] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9fd6af1f6d2445dfbf7c0c7df22ac206, de9196e9aa724b00b90c36fd643070bd; no voters: 
02:23:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.330896 21228 raft_consensus.cc:2454] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 0.
02:23:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.331408 21456 raft_consensus.cc:2781] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Leader pre-election won for term 1
02:23:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.331761 21456 raft_consensus.cc:482] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:23:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.332099 21456 raft_consensus.cc:3037] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Advancing to term 1
02:23:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.336831 21456 raft_consensus.cc:504] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.338418 21456 leader_election.cc:288] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 election: Requested vote from peers 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293), a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:21.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.339366 21076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "677c29065eaa44c49d687c21e5e9e194" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206"
02:23:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.339828 21076 raft_consensus.cc:3037] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Advancing to term 1
02:23:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.339633 21228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "677c29065eaa44c49d687c21e5e9e194" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3"
02:23:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.340200 21228 raft_consensus.cc:3037] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Advancing to term 1
02:23:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.344627 21076 raft_consensus.cc:2454] T 677c29065eaa44c49d687c21e5e9e194 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 1.
02:23:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.345785 21297 leader_election.cc:302] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9fd6af1f6d2445dfbf7c0c7df22ac206, de9196e9aa724b00b90c36fd643070bd; no voters: 
02:23:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.346531 21456 raft_consensus.cc:2781] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 1 FOLLOWER]: Leader election won for term 1
02:23:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.346868 21228 raft_consensus.cc:2454] T 677c29065eaa44c49d687c21e5e9e194 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 1.
02:23:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.349125 21456 raft_consensus.cc:686] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [term 1 LEADER]: Becoming Leader. State: Replica: de9196e9aa724b00b90c36fd643070bd, State: Running, Role: LEADER
02:23:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.350240 21456 consensus_queue.cc:237] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.363472 20861 catalog_manager.cc:5484] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd reported cstate change: term changed from 0 to 1, leader changed from <none> to de9196e9aa724b00b90c36fd643070bd (127.20.40.131). New cstate: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } health_report { overall_health: HEALTHY } } }
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.723779 20861 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37852:
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
02:23:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:23:21.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:21.725999 20861 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:23:21.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.753239 21456 consensus_queue.cc:1035] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:23:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.784416 21208 tablet_service.cc:1467] Processing CreateTablet for tablet e3ffd073d30645a99c26da8180fd7439 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:23:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.784420 21054 tablet_service.cc:1467] Processing CreateTablet for tablet e3ffd073d30645a99c26da8180fd7439 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:23:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.786175 21208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e3ffd073d30645a99c26da8180fd7439. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.786255 21054 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e3ffd073d30645a99c26da8180fd7439. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.804926 21484 consensus_queue.cc:1035] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [LEADER]: Connected to new peer: Peer: permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:23:21.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.805222 21355 tablet_service.cc:1467] Processing CreateTablet for tablet e3ffd073d30645a99c26da8180fd7439 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:23:21.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.806968 21355 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e3ffd073d30645a99c26da8180fd7439. 1 dirs total, 0 dirs full, 0 dirs failed
02:23:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.841601 21484 consensus_queue.cc:1035] T f6becb12d0fa43b39fece0887b13d671 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:23:21.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.866730 21493 tablet_bootstrap.cc:492] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: Bootstrap starting.
02:23:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.870057 21496 tablet_bootstrap.cc:492] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Bootstrap starting.
02:23:21.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.872661 21468 consensus_queue.cc:1035] T 677c29065eaa44c49d687c21e5e9e194 P de9196e9aa724b00b90c36fd643070bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:23:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.883320 21497 tablet_bootstrap.cc:492] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: Bootstrap starting.
02:23:21.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.904107 21493 tablet_bootstrap.cc:654] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: Neither blocks nor log segments found. Creating new log.
02:23:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.903061 21496 tablet_bootstrap.cc:654] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Neither blocks nor log segments found. Creating new log.
02:23:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.914276 21497 tablet_bootstrap.cc:654] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: Neither blocks nor log segments found. Creating new log.
02:23:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.917447 21496 tablet_bootstrap.cc:492] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: No bootstrap required, opened a new log
02:23:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.917973 21496 ts_tablet_manager.cc:1367] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Time spent bootstrapping tablet: real 0.050s	user 0.017s	sys 0.000s
02:23:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.919878 21493 tablet_bootstrap.cc:492] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: No bootstrap required, opened a new log
02:23:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.919965 21497 tablet_bootstrap.cc:492] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: No bootstrap required, opened a new log
02:23:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.920329 21493 ts_tablet_manager.cc:1367] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: Time spent bootstrapping tablet: real 0.054s	user 0.010s	sys 0.000s
02:23:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.920672 21497 ts_tablet_manager.cc:1367] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: Time spent bootstrapping tablet: real 0.038s	user 0.012s	sys 0.000s
02:23:21.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.920720 21496 raft_consensus.cc:348] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.921483 21496 raft_consensus.cc:374] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.921741 21496 raft_consensus.cc:729] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fd6af1f6d2445dfbf7c0c7df22ac206, State: Initialized, Role: FOLLOWER
02:23:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.922449 21496 consensus_queue.cc:260] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.922995 21493 raft_consensus.cc:348] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.923702 21493 raft_consensus.cc:374] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.923839 21486 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.924607 21486 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.924661 21496 ts_tablet_manager.cc:1398] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.923974 21493 raft_consensus.cc:729] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a4c37e2ef7c34a4db2b8c5114b6083c3, State: Initialized, Role: FOLLOWER
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.925827 21493 consensus_queue.cc:260] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.926259 21497 raft_consensus.cc:348] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.927740 21497 raft_consensus.cc:374] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:23:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.928017 21497 raft_consensus.cc:729] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: de9196e9aa724b00b90c36fd643070bd, State: Initialized, Role: FOLLOWER
02:23:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.930022 21493 ts_tablet_manager.cc:1398] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: Time spent starting tablet: real 0.009s	user 0.009s	sys 0.000s
02:23:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.929908 21454 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.930663 21454 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.930811 21497 consensus_queue.cc:260] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.932197 21456 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.933010 21456 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.935057 21497 ts_tablet_manager.cc:1398] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: Time spent starting tablet: real 0.014s	user 0.006s	sys 0.000s
02:23:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.981734 21456 raft_consensus.cc:482] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:23:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.982268 21456 raft_consensus.cc:504] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.984006 21456 leader_election.cc:288] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293), a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.984963 21076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e3ffd073d30645a99c26da8180fd7439" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" is_pre_election: true
02:23:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.985143 21228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e3ffd073d30645a99c26da8180fd7439" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" is_pre_election: true
02:23:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.985484 21076 raft_consensus.cc:2454] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 0.
02:23:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.985777 21228 raft_consensus.cc:2454] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 0.
02:23:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.986442 21297 leader_election.cc:302] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9fd6af1f6d2445dfbf7c0c7df22ac206, de9196e9aa724b00b90c36fd643070bd; no voters: 
02:23:21.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.987262 21456 raft_consensus.cc:2781] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Leader pre-election won for term 1
02:23:21.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.987599 21456 raft_consensus.cc:482] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:23:21.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.987943 21456 raft_consensus.cc:3037] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 0 FOLLOWER]: Advancing to term 1
02:23:21.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.993023 21456 raft_consensus.cc:504] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.994668 21456 leader_election.cc:288] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 election: Requested vote from peers 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293), a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.995844 21228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e3ffd073d30645a99c26da8180fd7439" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3"
02:23:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.995781 21076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e3ffd073d30645a99c26da8180fd7439" candidate_uuid: "de9196e9aa724b00b90c36fd643070bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206"
02:23:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.996416 21228 raft_consensus.cc:3037] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 0 FOLLOWER]: Advancing to term 1
02:23:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:21.996418 21076 raft_consensus.cc:3037] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 0 FOLLOWER]: Advancing to term 1
02:23:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.003505 21076 raft_consensus.cc:2454] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 1.
02:23:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.004715 21228 raft_consensus.cc:2454] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate de9196e9aa724b00b90c36fd643070bd in term 1.
02:23:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.006363 21297 leader_election.cc:302] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a4c37e2ef7c34a4db2b8c5114b6083c3, de9196e9aa724b00b90c36fd643070bd; no voters: 
02:23:22.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.007170 21456 raft_consensus.cc:2781] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 1 FOLLOWER]: Leader election won for term 1
02:23:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.007761 21456 raft_consensus.cc:686] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [term 1 LEADER]: Becoming Leader. State: Replica: de9196e9aa724b00b90c36fd643070bd, State: Running, Role: LEADER
02:23:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.008625 21456 consensus_queue.cc:237] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } }
02:23:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.017671 20861 catalog_manager.cc:5484] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd reported cstate change: term changed from 0 to 1, leader changed from <none> to de9196e9aa724b00b90c36fd643070bd (127.20.40.131). New cstate: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } health_report { overall_health: HEALTHY } } }
02:23:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.021148 21495 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: TxnStatusTablet state changed. Reason: New leader de9196e9aa724b00b90c36fd643070bd. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.021853 21495 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: This TxnStatusTablet replica's current role is: LEADER
02:23:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.025797 21503 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
02:23:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.026691 21503 txn_status_manager.cc:929] Loading transaction status metadata into memory...
02:23:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.101114 21228 raft_consensus.cc:1261] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3 [term 1 FOLLOWER]: Refusing update from remote peer de9196e9aa724b00b90c36fd643070bd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:23:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.101580 21076 raft_consensus.cc:1261] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206 [term 1 FOLLOWER]: Refusing update from remote peer de9196e9aa724b00b90c36fd643070bd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:23:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.102488 21495 consensus_queue.cc:1035] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:23:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.103247 21456 consensus_queue.cc:1035] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:23:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.120143 21454 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: TxnStatusTablet state changed. Reason: New leader de9196e9aa724b00b90c36fd643070bd. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.120991 21454 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.125299 21486 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: TxnStatusTablet state changed. Reason: New leader de9196e9aa724b00b90c36fd643070bd. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.126072 21486 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.129244 21489 mvcc.cc:205] Tried to move back new op lower bound from 7022715707791171584 to 7022715707434528768. Current Snapshot: MvccSnapshot[applied={T|T < 7022715707791171584}]
02:23:22.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.144701 21495 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.145490 21495 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: This TxnStatusTablet replica's current role is: LEADER
02:23:22.148 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
02:23:22.154 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table fa66dc7beb084580987b289db640f9aa
02:23:22.155 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.40.188:36585
02:23:22.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

02:23:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.156621 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 20819
02:23:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.158216 21456 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.159099 21456 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P de9196e9aa724b00b90c36fd643070bd: This TxnStatusTablet replica's current role is: LEADER
02:23:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.171259 21486 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.172135 21486 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P 9fd6af1f6d2445dfbf7c0c7df22ac206: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:22.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.178889 21454 tablet_replica.cc:439] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "de9196e9aa724b00b90c36fd643070bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" member_type: VOTER last_known_addr { host: "127.20.40.129" port: 45293 } } peers { permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" member_type: VOTER last_known_addr { host: "127.20.40.130" port: 42347 } } peers { permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" member_type: VOTER last_known_addr { host: "127.20.40.131" port: 44377 } } }
02:23:22.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.179822 21454 tablet_replica.cc:442] T e3ffd073d30645a99c26da8180fd7439 P a4c37e2ef7c34a4db2b8c5114b6083c3: This TxnStatusTablet replica's current role is: FOLLOWER
02:23:22.203 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.40.188:36585(127.20.40.188:36585)] handling channelInactive
02:23:22.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.188:36585(127.20.40.188:36585)] cleaning up while in state READY due to: connection closed
02:23:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.204131 21422 heartbeater.cc:644] Failed to heartbeat to 127.20.40.188:36585 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.20.40.188:36585 (error 108)
02:23:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.204486 21152 connection.cc:535] client connection to 127.20.40.188:36585 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:23:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.205344 21273 heartbeater.cc:644] Failed to heartbeat to 127.20.40.188:36585 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:23:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.206141 20999 connection.cc:535] client connection to 127.20.40.188:36585 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:23:22.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.206935 21121 heartbeater.cc:644] Failed to heartbeat to 127.20.40.188:36585 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:23:22.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:23:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.343520 21513 raft_consensus.cc:482] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader cf6ffc1cfe9b455ab3753f422a84e7db)
02:23:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.344084 21513 raft_consensus.cc:504] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:22.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.346072 21513 leader_election.cc:288] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585)
02:23:22.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.347057 20696 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e386f53f9dca4af0b5b66b91f47faa89" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" is_pre_election: true
02:23:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.351220 20759 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111) [suppressed 6 similar messages]
02:23:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.354745 20759 leader_election.cc:334] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.355073 20759 leader_election.cc:302] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e386f53f9dca4af0b5b66b91f47faa89; no voters: cf6ffc1cfe9b455ab3753f422a84e7db, ec33286a4c0b44fd90fcfc3e13d697cb
02:23:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.355648 21513 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:23:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.436537 21519 raft_consensus.cc:482] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 1 FOLLOWER]: Starting pre-election (detected failure of leader cf6ffc1cfe9b455ab3753f422a84e7db)
02:23:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.436985 21519 raft_consensus.cc:504] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.439005 21519 leader_election.cc:288] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575), cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585)
02:23:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.440049 20779 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e386f53f9dca4af0b5b66b91f47faa89" is_pre_election: true
02:23:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.440838 20779 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ec33286a4c0b44fd90fcfc3e13d697cb in term 1.
02:23:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.442130 20674 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111) [suppressed 12 similar messages]
02:23:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.444087 20674 leader_election.cc:302] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e386f53f9dca4af0b5b66b91f47faa89, ec33286a4c0b44fd90fcfc3e13d697cb; no voters: 
02:23:22.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.444947 21519 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 1 FOLLOWER]: Leader pre-election won for term 2
02:23:22.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.445299 21519 raft_consensus.cc:482] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 1 FOLLOWER]: Starting leader election (detected failure of leader cf6ffc1cfe9b455ab3753f422a84e7db)
02:23:22.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.445636 21519 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 1 FOLLOWER]: Advancing to term 2
02:23:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.446718 20674 leader_election.cc:334] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.452533 21519 raft_consensus.cc:504] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.454203 21519 leader_election.cc:288] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [CANDIDATE]: Term 2 election: Requested vote from peers e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575), cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585)
02:23:22.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.455112 20779 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e386f53f9dca4af0b5b66b91f47faa89"
02:23:22.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.455722 20779 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 1 FOLLOWER]: Advancing to term 2
02:23:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.457507 20674 leader_election.cc:334] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111)
02:23:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.464752 20779 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ec33286a4c0b44fd90fcfc3e13d697cb in term 2.
02:23:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.466012 20674 leader_election.cc:302] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e386f53f9dca4af0b5b66b91f47faa89, ec33286a4c0b44fd90fcfc3e13d697cb; no voters: cf6ffc1cfe9b455ab3753f422a84e7db
02:23:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.466931 21519 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 FOLLOWER]: Leader election won for term 2
02:23:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.468252 21519 raft_consensus.cc:686] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 LEADER]: Becoming Leader. State: Replica: ec33286a4c0b44fd90fcfc3e13d697cb, State: Running, Role: LEADER
02:23:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.469246 21519 consensus_queue.cc:237] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:22.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.473407 21522 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader ec33286a4c0b44fd90fcfc3e13d697cb. Latest consensus state: current_term: 2 leader_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:22.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.474324 21522 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: LEADER
02:23:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.476799 21523 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:23:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.486799 21523 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8]
02:23:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.488247 21523 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]
02:23:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.493541 21523 tablet_loader.cc:96] loaded metadata for tablet 677c29065eaa44c49d687c21e5e9e194 (table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa])
02:23:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.494623 21523 tablet_loader.cc:96] loaded metadata for tablet e3ffd073d30645a99c26da8180fd7439 (table kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8])
02:23:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.495558 21523 tablet_loader.cc:96] loaded metadata for tablet f6becb12d0fa43b39fece0887b13d671 (table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa])
02:23:22.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.496412 21523 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:23:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.499434 21523 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.499639 21523 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:23:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.503161 21523 catalog_manager.cc:1504] Loading token signing keys...
02:23:22.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.506211 21523 catalog_manager.cc:5868] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Loaded TSK: 0
02:23:22.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.507270 21523 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:23:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.555094 20779 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Refusing update from remote peer ec33286a4c0b44fd90fcfc3e13d697cb: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
02:23:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.556576 21522 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [LEADER]: Connected to new peer: Peer: permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
02:23:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.562608 21513 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ec33286a4c0b44fd90fcfc3e13d697cb. Latest consensus state: current_term: 2 leader_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.563498 21513 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: FOLLOWER
02:23:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.572749 21519 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.573784 21519 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: LEADER
02:23:22.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.575873 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:23:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.578167 21513 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.579051 21513 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: FOLLOWER
02:23:22.666 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.188:36585(127.20.40.188:36585)] connecting to peer
02:23:22.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585
02:23:22.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.188:36585(127.20.40.188:36585)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585
02:23:22.669 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.20.40.188:36585(127.20.40.188:36585) for tablet Kudu Master: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585
02:23:22.686 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.188:36585(127.20.40.188:36585)] connecting to peer
02:23:22.687 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585
02:23:22.687 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.188:36585(127.20.40.188:36585)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585
02:23:22.688 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.188:36585: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585
02:23:22.693 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:23:22.693 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=27), Traces: [0ms] sending RPC to server master-127.20.40.188:36585, [3ms] received response from server master-127.20.40.188:36585: Network error: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585, [3ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: Network error: Failed to connect to peer master-127.20.40.188:36585(127.20.40.188:36585): Connection refused: /127.20.40.188:36585, [25ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, deferred=Deferred@2009857234(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/391223744@79fbc9a6, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/888771670@507806b5)) after lookup
02:23:22.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:22.785982 20685 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49072:
02:23:22.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:23:22.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
02:23:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:23:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:22.793421 20685 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:23:23.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:23.103426 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:23:23.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:23.618654 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
02:23:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.123442 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
02:23:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.190086 21275 heartbeater.cc:503] Master 127.20.40.190:44353 was elected leader, sending a full tablet report...
02:23:24.220 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet f6becb12d0fa43b39fece0887b13d671 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
02:23:24.220 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 677c29065eaa44c49d687c21e5e9e194 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
02:23:24.220 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=f6becb12d0fa43b39fece0887b13d671@[9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293)[L],a4c37e2ef7c34a4db2b8c5114b6083c3(127.20.40.130:42347),de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=677c29065eaa44c49d687c21e5e9e194@[9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293),a4c37e2ef7c34a4db2b8c5114b6083c3(127.20.40.130:42347),de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
02:23:24.221 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.40.190:44353, [3ms] Sub RPC GetTableLocations: received response from server master-127.20.40.190:44353: OK, deferred=null) after lookup
02:23:24.221 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293)] connecting to peer
02:23:24.223 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293)] Successfully connected to peer
02:23:24.244 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x87b069b1, L:/127.0.0.1:60432 - R:/127.20.40.129:45293] using TOKEN/null
02:23:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.255856 21014 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.20.40.188:36585)
02:23:24.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.271999 21424 heartbeater.cc:503] Master 127.20.40.190:44353 was elected leader, sending a full tablet report...
02:23:24.377 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.40.188:36585
02:23:24.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.380220 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/logs
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/info.pb
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.188:36585
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.188
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46763
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:23:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:23:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.402700 21127 heartbeater.cc:503] Master 127.20.40.190:44353 was elected leader, sending a full tablet report...
02:23:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.635526 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
02:23:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.763128 21565 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.764221 21565 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.764905 21565 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.807410 21565 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:23:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.807783 21565 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:23:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.808054 21565 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.808334 21565 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:23:24.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.808669 21565 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:23:24.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.809015 21565 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:23:24.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.809473 21565 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43623/jwks.json
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.855204 21565 master_runner.cc:386] Master server non-default flags:
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:23:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.188:36585
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/info.pb
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.188
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46763
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/logs
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:24.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.856817 21565 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.858567 21565 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.880189 21572 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:25.182021 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
02:23:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:25.665634 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
02:23:26.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.880640 21573 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:26.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:24.888098 21565 server_base.cc:1034] running on GCE node
02:23:26.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:24.880997 21575 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.117115 21565 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.120504 21565 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.122004 21565 hybrid_clock.cc:648] HybridClock initialized: now 1714530206121942 us; error 65 us; skew 500 ppm
02:23:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.123037 21565 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:26.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.135891 21565 webserver.cc:460] Webserver started at http://127.20.40.188:46763/ using document root <none> and password file <none>
02:23:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.137375 21565 fs_manager.cc:359] Metadata directory not provided
02:23:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.137677 21565 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.148391 21565 fs_manager.cc:709] Time spent opening directory manager: real 0.007s	user 0.005s	sys 0.004s
02:23:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.155519 21585 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.157003 21565 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.000s
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.157407 21565 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf6ffc1cfe9b455ab3753f422a84e7db"
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:11 on dist-test-slave-hmt2"
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.159996 21565 fs_report.cc:383] FS layout report
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/wal
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/data
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.211441 20674 consensus_peers.cc:485] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb -> Peer cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585): Couldn't send request to peer cf6ffc1cfe9b455ab3753f422a84e7db. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.188:36585: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
02:23:26.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.239078 21565 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.240883 21565 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.241464 21565 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.362690 21565 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.188:36585
02:23:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.362877 21649 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.188:36585 every 8 connection(s)
02:23:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.366020 21565 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-2/data/info.pb
02:23:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.370019 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 21565
02:23:26.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:23:26.370 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.40.190:44353
02:23:26.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

02:23:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.371145 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 20654
02:23:26.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.380904 21650 sys_catalog.cc:263] Verifying existing consensus state
02:23:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.387893 21650 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Bootstrap starting.
02:23:26.412 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.40.190:44353(127.20.40.190:44353)] handling channelInactive
02:23:26.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state READY due to: connection closed
02:23:26.414 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:23:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.444025 21650 log.cc:824] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Log is configured to *not* fsync() on all Append() calls
02:23:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.450204 21121 heartbeater.cc:347] Connected to a master server at 127.20.40.188:36585
02:23:26.466 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.467 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.468 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.468 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.20.40.190:44353(127.20.40.190:44353) for tablet Kudu Master: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.483819 21650 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:23:26.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.484720 21650 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Bootstrap complete.
02:23:26.486 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.487 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.188:36585(127.20.40.188:36585)] connecting to peer
02:23:26.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.488 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.40.188:36585(127.20.40.188:36585)] Successfully connected to peer
02:23:26.491 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.40.188
02:23:26.503 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf888a690, L:/127.0.0.1:34014 - R:/127.20.40.188:36585] using SASL/PLAIN
02:23:26.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.188:36585: Catalog manager is not initialized. State: Starting
02:23:26.505 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.516726 21650 raft_consensus.cc:348] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:26.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.520551 21650 raft_consensus.cc:729] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: cf6ffc1cfe9b455ab3753f422a84e7db, State: Initialized, Role: FOLLOWER
02:23:26.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.521646 21650 consensus_queue.cc:260] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.524709 21656 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:26.526 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.525491 21656 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: FOLLOWER
02:23:26.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.526791 21650 sys_catalog.cc:564] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: configured and running, proceeding with master startup.
02:23:26.528 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.188:36585: Catalog manager is not initialized. State: Starting
02:23:26.528 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.547 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.548 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.548 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.548 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.557654 21667 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.558082 21667 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: loading cluster ID for follower catalog manager: success
02:23:26.558 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.567096 21667 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: acquiring CA information for follower catalog manager: success
02:23:26.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.573997 21667 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:23:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.550146 21565 master_runner.cc:186] Error getting master registration for 127.20.40.190:44353: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111), 
02:23:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.580667 21565 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:23:26.586 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.587 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.588 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.606 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.608 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.608 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.608 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.618336 21670 raft_consensus.cc:482] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader ec33286a4c0b44fd90fcfc3e13d697cb)
02:23:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.618865 21670 raft_consensus.cc:504] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.621026 21670 leader_election.cc:288] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585)
02:23:26.646 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.652 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.657699 20758 leader_election.cc:334] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111)
02:23:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.660902 21614 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e386f53f9dca4af0b5b66b91f47faa89" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" is_pre_election: true
02:23:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.662071 21614 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e386f53f9dca4af0b5b66b91f47faa89 in term 1.
02:23:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.664880 20759 leader_election.cc:302] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: cf6ffc1cfe9b455ab3753f422a84e7db, e386f53f9dca4af0b5b66b91f47faa89; no voters: ec33286a4c0b44fd90fcfc3e13d697cb
02:23:26.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.665781 21670 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:23:26.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.666178 21670 raft_consensus.cc:482] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Starting leader election (detected failure of leader ec33286a4c0b44fd90fcfc3e13d697cb)
02:23:26.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.666499 21670 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 2 FOLLOWER]: Advancing to term 3
02:23:26.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.672350 21670 raft_consensus.cc:504] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.674268 21670 leader_election.cc:288] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 3 election: Requested vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), cf6ffc1cfe9b455ab3753f422a84e7db (127.20.40.188:36585)
02:23:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.676337 21614 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e386f53f9dca4af0b5b66b91f47faa89" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db"
02:23:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.677039 21614 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 1 FOLLOWER]: Advancing to term 3
02:23:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.679706 20758 leader_election.cc:334] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111)
02:23:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.686182 21614 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e386f53f9dca4af0b5b66b91f47faa89 in term 3.
02:23:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.687330 20759 leader_election.cc:302] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: cf6ffc1cfe9b455ab3753f422a84e7db, e386f53f9dca4af0b5b66b91f47faa89; no voters: ec33286a4c0b44fd90fcfc3e13d697cb
02:23:26.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.688156 21670 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 3 FOLLOWER]: Leader election won for term 3
02:23:26.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.689718 21670 raft_consensus.cc:686] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [term 3 LEADER]: Becoming Leader. State: Replica: e386f53f9dca4af0b5b66b91f47faa89, State: Running, Role: LEADER
02:23:26.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.690817 21670 consensus_queue.cc:237] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.695957 21674 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e386f53f9dca4af0b5b66b91f47faa89. Latest consensus state: current_term: 3 leader_uuid: "e386f53f9dca4af0b5b66b91f47faa89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.696918 21674 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: LEADER
02:23:26.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.698973 21676 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:23:26.706 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.707 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.707556 21676 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8]
02:23:26.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.710673 21676 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]
02:23:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.717002 21676 tablet_loader.cc:96] loaded metadata for tablet 677c29065eaa44c49d687c21e5e9e194 (table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa])
02:23:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.718235 21676 tablet_loader.cc:96] loaded metadata for tablet e3ffd073d30645a99c26da8180fd7439 (table kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8])
02:23:26.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.719250 21676 tablet_loader.cc:96] loaded metadata for tablet f6becb12d0fa43b39fece0887b13d671 (table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa])
02:23:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.720204 21676 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:23:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.720831 20771 rpcz_store.cc:267] Call kudu.master.MasterService.ConnectToMaster from 127.0.0.1:43696 (request call id 8) took 13 ms (client timeout 10 ms). Trace:
02:23:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.721235 20771 rpcz_store.cc:269] 0501 02:23:26.707788 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:26.708055 (+   267us) service_pool.cc:225] Handling call
02:23:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:26.720594 (+ 12539us) inbound_call.cc:173] Queueing success response
02:23:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:23:26.722 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:26.723 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=256), Traces: [0ms] sending RPC to server master-127.20.40.190:44353, [2ms] received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [23ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [39ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [61ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [82ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [84ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [92ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [92ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [122ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [124ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [125ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [145ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [182ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [185ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [186ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [242ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [245ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [256ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, deferred=Deferred@1790153047(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/391223744@2cd4a9b2 -> (continuation of Deferred@359802653 after retry RPC after error@1517349046) -> (continuation of Deferred@742384518 after retry RPC after error@623859338) -> (continuation of Deferred@560060413 after retry RPC after error@2081013311) -> (continuation of Deferred@796475883 after retry RPC after error@1891490184) -> (continuation of Deferred@1598830173 after retry RPC after error@1854243789) -> (continuation of Deferred@186049007 after retry RPC after error@368646300), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/888771670@2b393aa0 -> (continuation of Deferred@359802653 after retry RPC after error@1517349046) -> (continuation of Deferred@742384518 after retry RPC after error@623859338) -> (continuation of Deferred@560060413 after retry RPC after error@2081013311) -> (continuation of Deferred@796475883 after retry RPC after error@1891490184) -> (continuation of Deferred@1598830173 after retry RPC after error@1854243789) -> (continuation of Deferred@186049007 after retry RPC after error@368646300)))
02:23:26.724 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=256), Traces: [0ms] sending RPC to server master-127.20.40.190:44353, [2ms] received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [23ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [39ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [61ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [82ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [84ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [92ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [92ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [122ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [124ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [125ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [145ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [182ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [185ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [186ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [242ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [245ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [256ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, deferred=Deferred@1790153047(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/391223744@2cd4a9b2 -> (continuation of Deferred@359802653 after retry RPC after error@1517349046) -> (continuation of Deferred@742384518 after retry RPC after error@623859338) -> (continuation of Deferred@560060413 after retry RPC after error@2081013311) -> (continuation of Deferred@796475883 after retry RPC after error@1891490184) -> (continuation of Deferred@1598830173 after retry RPC after error@1854243789) -> (continuation of Deferred@186049007 after retry RPC after error@368646300), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/888771670@2b393aa0 -> (continuation of Deferred@359802653 after retry RPC after error@1517349046) -> (continuation of Deferred@742384518 after retry RPC after error@623859338) -> (continuation of Deferred@560060413 after retry RPC after error@2081013311) -> (continuation of Deferred@796475883 after retry RPC after error@1891490184) -> (continuation of Deferred@1598830173 after retry RPC after error@1854243789) -> (continuation of Deferred@186049007 after retry RPC after error@368646300)))
02:23:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.724123 21676 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.724392 21676 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:23:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.728717 21676 catalog_manager.cc:1504] Loading token signing keys...
02:23:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.732035 21676 catalog_manager.cc:5868] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89: Loaded TSK: 0
02:23:26.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.733369 21676 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:23:26.746 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:26.750 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353
02:23:26.757 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:23:26.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=14), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [4ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Network error: Failed to connect to peer master-127.20.40.190:44353(127.20.40.190:44353): Connection refused: /127.20.40.190:44353, [11ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: OK, [13ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, deferred=null) after lookup
02:23:26.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.790099 21614 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Refusing update from remote peer e386f53f9dca4af0b5b66b91f47faa89: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
02:23:26.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.792176 21674 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
02:23:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.823408 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:23:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.854317 21656 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: New leader e386f53f9dca4af0b5b66b91f47faa89. Latest consensus state: current_term: 3 leader_uuid: "e386f53f9dca4af0b5b66b91f47faa89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.855435 21656 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: FOLLOWER
02:23:26.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.857933 21688 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "e386f53f9dca4af0b5b66b91f47faa89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:26.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.858886 21688 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: FOLLOWER
02:23:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.866556 21670 sys_catalog.cc:455] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "e386f53f9dca4af0b5b66b91f47faa89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.867461 21670 sys_catalog.cc:458] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 [sys.catalog]: This master's current role is: LEADER
02:23:26.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.872064 21688 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "e386f53f9dca4af0b5b66b91f47faa89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:26.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.872889 21688 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: FOLLOWER
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:26.892562 20771 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41390:
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:23:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.926820 20771 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:23:26.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.959247 20802 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:43696 (request call id 10) took 197 ms (client timeout 236 ms). Trace:
02:23:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:26.959692 20802 rpcz_store.cc:269] 0501 02:23:26.761348 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:26.761561 (+   213us) service_pool.cc:225] Handling call
02:23:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:26.959001 (+197440us) inbound_call.cc:173] Queueing success response
02:23:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":1260,"init.queue_time_us":13126,"init.run_cpu_time_us":38191,"init.run_wall_time_us":158941,"thread_start_us":51982,"threads_started":5}
02:23:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.253028 21273 heartbeater.cc:347] Connected to a master server at 127.20.40.188:36585
02:23:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.261101 21604 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" instance_seqno: 1714530198558612) as {username='slave'} at 127.20.40.130:58901; Asking this server to re-register.
02:23:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.262850 21273 heartbeater.cc:464] Registering TS with master...
02:23:27.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.267349 21604 ts_manager.cc:194] Registered new tserver with Master: a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:27.320999 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:23:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.353219 21422 heartbeater.cc:347] Connected to a master server at 127.20.40.188:36585
02:23:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.358816 21604 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" instance_seqno: 1714530200579070) as {username='slave'} at 127.20.40.131:33933; Asking this server to re-register.
02:23:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.360782 21422 heartbeater.cc:464] Registering TS with master...
02:23:27.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.364575 21604 ts_manager.cc:194] Registered new tserver with Master: de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377)
02:23:27.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:27.417430 20758 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111) [suppressed 8 similar messages]
02:23:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.465016 21604 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" instance_seqno: 1714530195691859) as {username='slave'} at 127.20.40.129:60277; Asking this server to re-register.
02:23:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.466872 21121 heartbeater.cc:464] Registering TS with master...
02:23:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:27.471189 21604 ts_manager.cc:194] Registered new tserver with Master: 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293)
02:23:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:27.799883 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
02:23:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:28.213958 21274 heartbeater.cc:503] Master 127.20.40.189:43575 was elected leader, sending a full tablet report...
02:23:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:28.226809 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
02:23:28.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:28.333122 21423 heartbeater.cc:503] Master 127.20.40.189:43575 was elected leader, sending a full tablet report...
02:23:28.421 [DEBUG - Time-limited test] (Connection.java:748) [peer de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)] connecting to peer
02:23:28.422 [DEBUG - kudu-nio-0] (Connection.java:757) [peer de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)] Successfully connected to peer
02:23:28.438 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc7b6295c, L:/127.0.0.1:59162 - R:/127.20.40.131:44377] using TOKEN/null
02:23:28.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:28.437072 21122 heartbeater.cc:503] Master 127.20.40.189:43575 was elected leader, sending a full tablet report...
02:23:28.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:28.467491 21320 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.20.40.188:36585)
02:23:28.575 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.40.190:44353
02:23:28.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:28.578483 20642 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/logs
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/info.pb
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.190:44353
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.190
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42245
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:23:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:23:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:28.736022 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
02:23:29.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.006364 21718 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:23:29.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.007040 21718 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:23:29.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.007580 21718 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:23:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.046725 21718 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:23:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.047180 21718 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:23:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.047734 21718 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:23:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.048170 21718 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:23:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.048493 21718 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:23:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.048801 21718 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:23:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.049062 21718 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43623/jwks.json
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:29.093094 21718 master_runner.cc:386] Master server non-default flags:
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.40.148:40563
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.40.190:44353
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:23:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43623/jwks.json
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/info.pb
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.40.190
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42245
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/logs
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:23:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:29.094568 21718 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:23:29.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:29.096287 21718 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:23:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.117200 21726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.272156 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
02:23:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.762615 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
02:23:30.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.269294 20758 consensus_peers.cc:485] T 00000000000000000000000000000000 P e386f53f9dca4af0b5b66b91f47faa89 -> Peer ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353): Couldn't send request to peer ec33286a4c0b44fd90fcfc3e13d697cb. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.190:44353: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
02:23:30.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.118424 21727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:30.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:29.122581 21729 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:23:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.343681 21728 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:23:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.343737 21718 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:23:30.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.348332 21718 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:23:30.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.354344 21718 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:23:30.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.355736 21718 hybrid_clock.cc:648] HybridClock initialized: now 1714530210355704 us; error 37 us; skew 500 ppm
02:23:30.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.356578 21718 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:23:30.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.364454 21718 webserver.cc:460] Webserver started at http://127.20.40.190:42245/ using document root <none> and password file <none>
02:23:30.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.365590 21718 fs_manager.cc:359] Metadata directory not provided
02:23:30.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.365813 21718 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:23:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.377100 21718 fs_manager.cc:709] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.000s
02:23:30.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.384387 21741 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:23:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.385713 21718 fs_manager.cc:725] Time spent opening block manager: real 0.006s	user 0.000s	sys 0.002s
02:23:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.386096 21718 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data,/tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec33286a4c0b44fd90fcfc3e13d697cb"
02:23:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:23:08 on dist-test-slave-hmt2"
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.388631 21718 fs_report.cc:383] FS layout report
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/wal
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/data
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:23:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:23:30.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.462256 21718 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:23:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.463908 21718 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:23:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.464359 21718 kserver.cc:163] Server-wide thread pool size limit: 3276
02:23:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.560251 21718 rpc_server.cc:307] RPC server started. Bound to: 127.20.40.190:44353
02:23:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.560319 21803 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.40.190:44353 every 8 connection(s)
02:23:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.563191 21718 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGqEZ7/test-tmp/mini-kudu-cluster5585314079014412692/master-0/data/info.pb
02:23:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.569648 20642 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu as pid 21718
02:23:30.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:23:30.570 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.40.189:43575
02:23:30.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

02:23:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.571491 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 20734
02:23:30.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.575804 21804 sys_catalog.cc:263] Verifying existing consensus state
02:23:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.581750 21804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Bootstrap starting.
02:23:30.609 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:23:30.607 [ERROR - kudu-nio-1] (Connection.java:452) [peer master-127.20.40.189:43575(127.20.40.189:43575)] unexpected exception from downstream on [id: 0x921944e0, L:/127.0.0.1:43696 - R:/127.20.40.189:43575]
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
	at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
	at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:256) ~[netty-buffer-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:357) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
	at sun.nio.ch.IOUtil.read(IOUtil.java:192)
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
	at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:256)
	at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
	at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:357)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

02:23:30.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state READY due to: [peer master-127.20.40.189:43575(127.20.40.189:43575)] unexpected exception from downstream on [id: 0x921944e0, L:/127.0.0.1:43696 - R:/127.20.40.189:43575]
02:23:30.627 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.20.40.189:43575(127.20.40.189:43575) for tablet Kudu Master: [peer master-127.20.40.189:43575(127.20.40.189:43575)] unexpected exception from downstream on [id: 0x921944e0, L:/127.0.0.1:43696 - R:/127.20.40.189:43575]
02:23:30.628 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.40.189:43575(127.20.40.189:43575)] handling channelInactive
02:23:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.634198 21804 log.cc:824] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Log is configured to *not* fsync() on all Append() calls
02:23:30.646 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.190:44353(127.20.40.190:44353)] connecting to peer
02:23:30.647 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.647 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.40.190:44353(127.20.40.190:44353)] Successfully connected to peer
02:23:30.648 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.648 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.648 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.660 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.40.190
02:23:30.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.673802 21117 debug-util.cc:398] Leaking SignalData structure 0x7b080005b0e0 after lost signal to thread 20959
02:23:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.674966 21117 debug-util.cc:398] Leaking SignalData structure 0x7b08000d8700 after lost signal to thread 21120
02:23:30.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.676862 21804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:23:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.678036 21804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: Bootstrap complete.
02:23:30.679 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xdc8e50c0, L:/127.0.0.1:51872 - R:/127.20.40.190:44353] using SASL/PLAIN
02:23:30.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Catalog manager is not initialized. State: Starting
02:23:30.686 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.701556 21804 raft_consensus.cc:348] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:30.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.704034 21804 raft_consensus.cc:729] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: ec33286a4c0b44fd90fcfc3e13d697cb, State: Initialized, Role: FOLLOWER
02:23:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.704787 21804 consensus_queue.cc:260] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:30.706 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.708 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Catalog manager is not initialized. State: Starting
02:23:30.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.710783 21808 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:30.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.713063 21808 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:30.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.713704 21804 sys_catalog.cc:564] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: configured and running, proceeding with master startup.
02:23:30.726 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.731 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.190:44353: Catalog manager is not initialized. State: Starting
02:23:30.734 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.770 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.774 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.780586 21819 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:30.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.780957 21819 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: loading cluster ID for follower catalog manager: success
02:23:30.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.794562 21819 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: acquiring CA information for follower catalog manager: success
02:23:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.802083 21819 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:23:30.807 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.809911 21823 raft_consensus.cc:482] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Starting pre-election (detected failure of leader e386f53f9dca4af0b5b66b91f47faa89)
02:23:30.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.810506 21823 raft_consensus.cc:504] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:30.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.814759 21823 leader_election.cc:288] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575)
02:23:30.816 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.827982 21589 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:30.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.841605 21589 leader_election.cc:334] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:30.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.843070 21769 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" is_pre_election: true
02:23:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.848119 21769 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cf6ffc1cfe9b455ab3753f422a84e7db in term 2.
02:23:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.790102 21718 master_runner.cc:186] Error getting master registration for 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111), 
02:23:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.850143 21588 leader_election.cc:302] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: cf6ffc1cfe9b455ab3753f422a84e7db, ec33286a4c0b44fd90fcfc3e13d697cb; no voters: e386f53f9dca4af0b5b66b91f47faa89
02:23:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.850801 21718 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:23:30.851 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=264), Traces: [0ms] sending RPC to server master-127.20.40.189:43575, [40ms] received response from server master-127.20.40.189:43575: Network error: [peer master-127.20.40.189:43575(127.20.40.189:43575)] unexpected exception from downstream on [id: 0x921944e0, L:/127.0.0.1:43696 - R:/127.20.40.189:43575], [41ms] delaying RPC due to: Network error: [peer master-127.20.40.189:43575(127.20.40.189:43575)] unexpected exception from downstream on [id: 0x921944e0, L:/127.0.0.1:43696 - R:/127.20.40.189:43575], [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [64ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [99ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [100ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [122ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [123ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [124ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [145ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [146ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [148ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [149ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [182ms] refreshing cache from master, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [185ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [187ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [188ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [225ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [229ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [230ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [230ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, deferred=Deferred@1815676858(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/391223744@69656b0f -> (continuation of Deferred@369906492 after retry RPC after error@1282682472) -> (continuation of Deferred@1758786699 after retry RPC after error@715900813) -> (continuation of Deferred@1590359127 after retry RPC after error@857773704) -> (continuation of Deferred@2082430155 after retry RPC after error@142375786) -> (continuation of Deferred@18701950 after retry RPC after error@910040067), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/888771670@30c1f70b -> (continuation of Deferred@369906492 after retry RPC after error@1282682472) -> (continuation of Deferred@1758786699 after retry RPC after error@715900813) -> (continuation of Deferred@1590359127 after retry RPC after error@857773704) -> (continuation of Deferred@2082430155 after retry RPC after error@142375786) -> (continuation of Deferred@18701950 after retry RPC after error@910040067)))
02:23:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.851421 21823 raft_consensus.cc:2781] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Leader pre-election won for term 4
02:23:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.851980 21823 raft_consensus.cc:482] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Starting leader election (detected failure of leader e386f53f9dca4af0b5b66b91f47faa89)
02:23:30.851 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
	at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
	at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
	at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
	at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
	at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
	at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
	at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
	at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
	at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
	at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
	at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
	at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
	at java.lang.Thread.run(Thread.java:748)

02:23:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.852419 21823 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 3 FOLLOWER]: Advancing to term 4
02:23:30.867 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.861284 21823 raft_consensus.cc:504] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.864682 21769 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb"
02:23:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.865357 21769 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 2 FOLLOWER]: Advancing to term 4
02:23:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:30.881405 21589 leader_election.cc:334] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111)
02:23:30.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.885655 21769 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cf6ffc1cfe9b455ab3753f422a84e7db in term 4.
02:23:30.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.878955 21823 leader_election.cc:288] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 4 election: Requested vote from peers ec33286a4c0b44fd90fcfc3e13d697cb (127.20.40.190:44353), e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575)
02:23:30.894 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.894701 21588 leader_election.cc:302] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: cf6ffc1cfe9b455ab3753f422a84e7db, ec33286a4c0b44fd90fcfc3e13d697cb; no voters: e386f53f9dca4af0b5b66b91f47faa89
02:23:30.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.897172 21823 raft_consensus.cc:2781] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 4 FOLLOWER]: Leader election won for term 4
02:23:30.906 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.907 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.908 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.918408 21823 raft_consensus.cc:686] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [term 4 LEADER]: Becoming Leader. State: Replica: cf6ffc1cfe9b455ab3753f422a84e7db, State: Running, Role: LEADER
02:23:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.920441 21823 consensus_queue.cc:237] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } }
02:23:30.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.942477 21827 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: New leader cf6ffc1cfe9b455ab3753f422a84e7db. Latest consensus state: current_term: 4 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:30.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.943411 21827 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: LEADER
02:23:30.946 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.966 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.967209 21829 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:23:30.968 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.968 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.968 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:30.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.977035 21829 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8]
02:23:30.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.979022 21829 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa]
02:23:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.986649 21829 tablet_loader.cc:96] loaded metadata for tablet 677c29065eaa44c49d687c21e5e9e194 (table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa])
02:23:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.988211 21829 tablet_loader.cc:96] loaded metadata for tablet e3ffd073d30645a99c26da8180fd7439 (table kudu_system.kudu_transactions [id=c82d8600895c47a7986a5f51bfd288e8])
02:23:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.989668 21829 tablet_loader.cc:96] loaded metadata for tablet f6becb12d0fa43b39fece0887b13d671 (table txn_manager_fallback_rolling [id=fa66dc7beb084580987b289db640f9aa])
02:23:30.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.991108 21829 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:23:30.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.995666 21829 catalog_manager.cc:1259] Loaded cluster ID: 244ad1c4cf794cac8ec542fbd2d766d6
02:23:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:30.996003 21829 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:23:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.004773 21829 catalog_manager.cc:1504] Loading token signing keys...
02:23:31.007 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:31.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:31.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:31.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.008374 21769 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [term 4 FOLLOWER]: Refusing update from remote peer cf6ffc1cfe9b455ab3753f422a84e7db: Log matching property violated. Preceding OpId in replica: term: 2 index: 12. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
02:23:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.012149 21829 catalog_manager.cc:5868] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db: Loaded TSK: 0
02:23:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.013347 21827 consensus_queue.cc:1035] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [LEADER]: Connected to new peer: Peer: permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
02:23:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.013903 21829 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:23:31.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:31.018 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585; will retry
02:23:31.046 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.20.40.189:43575(127.20.40.189:43575)] connecting to peer
02:23:31.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:31.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.189:43575(127.20.40.189:43575)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:31.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.40.189:43575: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575
02:23:31.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.053042 21589 consensus_peers.cc:485] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db -> Peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Couldn't send request to peer e386f53f9dca4af0b5b66b91f47faa89. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:23:31.061 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:23:31.063 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=196), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [2ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [10ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [28ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [42ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [45ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [75ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [75ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [82ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [84ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [85ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [102ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [105ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [126ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [150ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [152ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [152ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [183ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, deferred=Deferred@401444204(state=PENDING, result=null, callback=(continuation of Deferred@1339934056 after retry RPC after error@335619481) -> (continuation of Deferred@719786473 after retry RPC after error@1360690996) -> (continuation of Deferred@1892484858 after retry RPC after error@1845348503) -> (continuation of Deferred@2139616408 after retry RPC after error@2112276355) -> (continuation of Deferred@1694823443 after retry RPC after error@1715347796), errback=(continuation of Deferred@1339934056 after retry RPC after error@335619481) -> (continuation of Deferred@719786473 after retry RPC after error@1360690996) -> (continuation of Deferred@1892484858 after retry RPC after error@1845348503) -> (continuation of Deferred@2139616408 after retry RPC after error@2112276355) -> (continuation of Deferred@1694823443 after retry RPC after error@1715347796))) after lookup
02:23:31.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.067440 21832 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.070034 21808 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: New leader cf6ffc1cfe9b455ab3753f422a84e7db. Latest consensus state: current_term: 4 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.070775 21808 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.074569 21808 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.075525 21808 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.068894 21832 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec33286a4c0b44fd90fcfc3e13d697cb [sys.catalog]: This master's current role is: FOLLOWER
02:23:31.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.087188 21827 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec33286a4c0b44fd90fcfc3e13d697cb" member_type: VOTER last_known_addr { host: "127.20.40.190" port: 44353 } } peers { permanent_uuid: "e386f53f9dca4af0b5b66b91f47faa89" member_type: VOTER last_known_addr { host: "127.20.40.189" port: 43575 } } peers { permanent_uuid: "cf6ffc1cfe9b455ab3753f422a84e7db" member_type: VOTER last_known_addr { host: "127.20.40.188" port: 36585 } } }
02:23:31.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.088100 21827 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db [sys.catalog]: This master's current role is: LEADER
02:23:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.118148 21635 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:34014 (request call id 19) took 53 ms (client timeout 53 ms). Trace:
02:23:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.118616 21635 rpcz_store.cc:269] 0501 02:23:31.064717 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:31.064909 (+   192us) service_pool.cc:225] Handling call
02:23:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:31.091073 (+ 26164us) spinlock_profiling.cc:243] Waited 4.08 ms on lock 0x7b54000203d4. stack: 00007f4919ee6ba3 00007f4919ee6aa0 00007f4919a6e584 00000000004e42e4 00000000004e4279 00007f4920df66b1 00007f4920e7a239 00007f4922e03a26 00007f4922ec6dac 00007f4922e02082 00007f4922e01c94 00007f4922e072f1 00007f4922e072a9 00007f4922e07239 00007f4922e07201 00007f4922e064fd
02:23:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:31.117860 (+ 26787us) inbound_call.cc:173] Queueing success response
02:23:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":19647,"spinlock_wait_cycles":9379712,"thread_start_us":35750,"threads_started":3}
02:23:31.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=255), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [2ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [10ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [28ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [42ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [45ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [75ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [75ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [82ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [84ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [85ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [102ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [105ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [126ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [150ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [152ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [152ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [183ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [197ms] sending RPC to server master-127.20.40.188:36585, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [255ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@401444204(state=PENDING, result=null, callback=(continuation of Deferred@1339934056 after retry RPC after error@335619481) -> (continuation of Deferred@719786473 after retry RPC after error@1360690996) -> (continuation of Deferred@1892484858 after retry RPC after error@1845348503) -> (continuation of Deferred@2139616408 after retry RPC after error@2112276355) -> (continuation of Deferred@1694823443 after retry RPC after error@1715347796) -> (continuation of Deferred@643643967 after retry RPC@1833712552), errback=(continuation of Deferred@1339934056 after retry RPC after error@335619481) -> (continuation of Deferred@719786473 after retry RPC after error@1360690996) -> (continuation of Deferred@1892484858 after retry RPC after error@1845348503) -> (continuation of Deferred@2139616408 after retry RPC after error@2112276355) -> (continuation of Deferred@1694823443 after retry RPC after error@1715347796) -> (continuation of Deferred@643643967 after retry RPC@1833712552)))
02:23:31.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=255), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [2ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [10ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [28ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [42ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [45ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [75ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [75ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [82ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [84ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [85ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [102ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [105ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [126ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [150ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [152ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [152ms] delaying RPC due to: Service unavailable: Master config (127.20.40.190:44353,127.20.40.189:43575,127.20.40.188:36585) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.190:44353, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.189:43575, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.40.188:36585, [183ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.189:43575: Network error: Failed to connect to peer master-127.20.40.189:43575(127.20.40.189:43575): Connection refused: /127.20.40.189:43575, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.188:36585: OK, [197ms] sending RPC to server master-127.20.40.188:36585, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.40.190:44353: OK, [255ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@401444204(state=PENDING, result=null, callback=(continuation of Deferred@1339934056 after retry RPC after error@335619481) -> (continuation of Deferred@719786473 after retry RPC after error@1360690996) -> (continuation of Deferred@1892484858 after retry RPC after error@1845348503) -> (continuation of Deferred@2139616408 after retry RPC after error@2112276355) -> (continuation of Deferred@1694823443 after retry RPC after error@1715347796) -> (continuation of Deferred@643643967 after retry RPC@1833712552), errback=(continuation of Deferred@1339934056 after retry RPC after error@335619481) -> (continuation of Deferred@719786473 after retry RPC after error@1360690996) -> (continuation of Deferred@1892484858 after retry RPC after error@1845348503) -> (continuation of Deferred@2139616408 after retry RPC after error@2112276355) -> (continuation of Deferred@1694823443 after retry RPC after error@1715347796) -> (continuation of Deferred@643643967 after retry RPC@1833712552)))
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.241043 21604 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34048:
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
02:23:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:23:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.252686 21604 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:23:31.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.270648 21303 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.101s from last keepalive heartbeat (effective timeout is 1.000s)
02:23:31.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.286144 21275 heartbeater.cc:347] Connected to a master server at 127.20.40.190:44353
02:23:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.301734 21320 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.20.40.189:43575)
02:23:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.309252 21759 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a4c37e2ef7c34a4db2b8c5114b6083c3" instance_seqno: 1714530198558612) as {username='slave'} at 127.20.40.130:51925; Asking this server to re-register.
02:23:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.316423 21275 heartbeater.cc:464] Registering TS with master...
02:23:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.327639 21759 ts_manager.cc:194] Registered new tserver with Master: a4c37e2ef7c34a4db2b8c5114b6083c3 (127.20.40.130:42347)
02:23:31.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.341238 21635 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:34014 (request call id 20) took 213 ms (client timeout 250 ms). Trace:
02:23:31.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.341665 21635 rpcz_store.cc:269] 0501 02:23:31.127915 (+     0us) service_pool.cc:168] Inserting onto call queue
02:23:31.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:31.128122 (+   207us) service_pool.cc:225] Handling call
02:23:31.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 02:23:31.341163 (+213041us) inbound_call.cc:173] Queueing success response
02:23:31.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:23:31.343 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1714530202 last_transition_timestamp: 1714530211
02:23:31.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.359418 21843 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
02:23:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.405932 21424 heartbeater.cc:347] Connected to a master server at 127.20.40.190:44353
02:23:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.410916 21759 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "de9196e9aa724b00b90c36fd643070bd" instance_seqno: 1714530200579070) as {username='slave'} at 127.20.40.131:59015; Asking this server to re-register.
02:23:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.413901 21424 heartbeater.cc:464] Registering TS with master...
02:23:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.415904 21847 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
02:23:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.420118 21759 ts_manager.cc:194] Registered new tserver with Master: de9196e9aa724b00b90c36fd643070bd (127.20.40.131:44377)
02:23:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.519205 21127 heartbeater.cc:347] Connected to a master server at 127.20.40.190:44353
02:23:31.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.523890 21759 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9fd6af1f6d2445dfbf7c0c7df22ac206" instance_seqno: 1714530195691859) as {username='slave'} at 127.20.40.129:56309; Asking this server to re-register.
02:23:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.530979 21127 heartbeater.cc:464] Registering TS with master...
02:23:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.536903 21759 ts_manager.cc:194] Registered new tserver with Master: 9fd6af1f6d2445dfbf7c0c7df22ac206 (127.20.40.129:45293)
02:23:31.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.593613 21565 master_runner.cc:186] Error getting master registration for 127.20.40.189:43575: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111), 
02:23:31.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:31.594305 21565 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:23:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:31.616693 21589 consensus_peers.cc:485] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db -> Peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Couldn't send request to peer e386f53f9dca4af0b5b66b91f47faa89. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:23:32.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:32.137601 21589 consensus_peers.cc:485] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db -> Peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Couldn't send request to peer e386f53f9dca4af0b5b66b91f47faa89. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
02:23:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.289809 21273 heartbeater.cc:503] Master 127.20.40.188:36585 was elected leader, sending a full tablet report...
02:23:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.379426 21422 heartbeater.cc:503] Master 127.20.40.188:36585 was elected leader, sending a full tablet report...
02:23:32.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.497130 21121 heartbeater.cc:503] Master 127.20.40.188:36585 was elected leader, sending a full tablet report...
02:23:32.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:23:32.591326 21589 consensus_peers.cc:485] T 00000000000000000000000000000000 P cf6ffc1cfe9b455ab3753f422a84e7db -> Peer e386f53f9dca4af0b5b66b91f47faa89 (127.20.40.189:43575): Couldn't send request to peer e386f53f9dca4af0b5b66b91f47faa89. Status: Network error: Client connection negotiation failed: client connection to 127.20.40.189:43575: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
02:23:32.619 [DEBUG - main] (Connection.java:575) [peer master-127.20.40.189:43575(127.20.40.189:43575)] disconnecting while in state TERMINATED
02:23:32.620 [DEBUG - main] (Connection.java:575) [peer master-127.20.40.188:36585(127.20.40.188:36585)] disconnecting while in state READY
02:23:32.620 [DEBUG - main] (Connection.java:575) [peer de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)] disconnecting while in state READY
02:23:32.620 [DEBUG - main] (Connection.java:575) [peer 9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293)] disconnecting while in state READY
02:23:32.620 [DEBUG - main] (Connection.java:575) [peer master-127.20.40.190:44353(127.20.40.190:44353)] disconnecting while in state READY
02:23:32.621 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293)] handling channelInactive
02:23:32.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9fd6af1f6d2445dfbf7c0c7df22ac206(127.20.40.129:45293)] cleaning up while in state READY due to: connection closed
02:23:32.621 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.40.190:44353(127.20.40.190:44353)] handling channelInactive
02:23:32.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.40.190:44353(127.20.40.190:44353)] cleaning up while in state READY due to: connection closed
02:23:32.624 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:23:32.629 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.40.188:36585(127.20.40.188:36585)] handling channelInactive
02:23:32.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.40.188:36585(127.20.40.188:36585)] cleaning up while in state READY due to: connection closed
02:23:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.630314 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 20952
02:23:32.634 [DEBUG - kudu-nio-0] (Connection.java:239) [peer de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)] handling channelInactive
02:23:32.634 [DEBUG - kudu-nio-0] (Connection.java:709) [peer de9196e9aa724b00b90c36fd643070bd(127.20.40.131:44377)] cleaning up while in state READY due to: connection closed
02:23:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.680292 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 21124
02:23:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.718503 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 21278
02:23:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.750481 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 21718
02:23:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:23:32.786167 20642 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGqEZ7/build/tsan/bin/kudu with pid 21565
02:23:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:23:33Z chronyd exiting
02:23:34.987 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
	at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)