Unable to diagnose

20:08:23.770 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:08:23.770 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task6wKKBW/test-tmp
20:08:23.771 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
20:08:23.771 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task6wKKBW/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
20:08:23.772 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343"
  extra_master_flags: "--txn_manager_enabled"
  extra_master_flags: "--raft_heartbeat_interval_ms=100"
  extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
  extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
  extra_tserver_flags: "--enable_txn_system_client_init=true"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

20:08:24.069 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:08:24.069 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

20:08:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.071655 21550 mini_oidc.cc:152] Starting JWKS server
20:08:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.073819 21550 webserver.cc:460] Webserver started at http://127.0.0.1:45863/ using document root <none> and password file <none>
20:08:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.073983 21550 mini_oidc.cc:189] Starting OIDC Discovery server
20:08:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.074455 21550 webserver.cc:460] Webserver started at http://127.0.0.1:34561/ using document root <none> and password file <none>
20:08:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-06T20:08:24Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:08:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-06T20:08:24Z Disabled control of system clock
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.101198 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/logs
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/info.pb
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.190:36769
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.190
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:08:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.395187 21561 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.395507 21561 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.395628 21561 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.402737 21561 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:08:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.402882 21561 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:08:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.402937 21561 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.403000 21561 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.403057 21561 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:08:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.403129 21561 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.403180 21561 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45863/jwks.json
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.411377 21561 master_runner.cc:386] Master server non-default flags:
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.190:36769
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/info.pb
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.190
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/logs
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.412986 21561 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.414628 21561 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:24.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.425088 21566 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:24.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.425328 21567 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.426106 21569 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.426834 21561 server_base.cc:1034] running on GCE node
20:08:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.427567 21561 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.428891 21561 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.430091 21561 hybrid_clock.cc:648] HybridClock initialized: now 1715026104430051 us; error 35 us; skew 500 ppm
20:08:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.430459 21561 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.432636 21561 webserver.cc:460] Webserver started at http://127.21.11.190:36499/ using document root <none> and password file <none>
20:08:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.433305 21561 fs_manager.cc:359] Metadata directory not provided
20:08:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.433409 21561 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.433689 21561 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.436076 21561 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/instance:
20:08:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "279092f506ba4af8b3d5dec9ce4ba0aa"
20:08:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.436806 21561 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal/instance:
20:08:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "279092f506ba4af8b3d5dec9ce4ba0aa"
20:08:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.441704 21561 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.000s
20:08:24.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.444859 21575 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.446071 21561 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.446208 21561 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "279092f506ba4af8b3d5dec9ce4ba0aa"
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.446415 21561 fs_report.cc:383] FS layout report
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/data
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:24.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:24.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.466440 21561 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.467293 21561 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.467566 21561 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:24.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.486609 21561 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.190:36769
20:08:24.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.486625 21636 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.190:36769 every 8 connection(s)
20:08:24.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.487812 21561 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/info.pb
20:08:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.491264 21637 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:24.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.494782 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 21561
20:08:24.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.495014 21550 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal/instance
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.497112 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/logs
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/info.pb
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.189:34501
20:08:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.189
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.496938 21637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } has no permanent_uuid. Determining permanent_uuid...
20:08:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.512691 21637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } has no permanent_uuid. Determining permanent_uuid...
20:08:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.513629 21576 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.515005 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.189:34501: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.564330 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } attempt: 1
20:08:24.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.566219 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.189:34501: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.634553 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } attempt: 2
20:08:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.636368 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.189:34501: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.727667 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } attempt: 3
20:08:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.729645 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.189:34501: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.803624 21640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:24.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.803916 21640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:24.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.804003 21640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.810827 21640 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.810937 21640 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.810972 21640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.811017 21640 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.811054 21640 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.811101 21640 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.811131 21640 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45863/jwks.json
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.819854 21640 master_runner.cc:386] Master server non-default flags:
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.189:34501
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/info.pb
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.189
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/logs
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.821219 21640 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:24.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.822954 21640 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.832693 21647 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.833295 21650 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.833683 21648 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.834466 21640 server_base.cc:1034] running on GCE node
20:08:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.835176 21640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.836434 21640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:24.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.837620 21640 hybrid_clock.cc:648] HybridClock initialized: now 1715026104837579 us; error 32 us; skew 500 ppm
20:08:24.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.837939 21640 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:24.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.840095 21640 webserver.cc:460] Webserver started at http://127.21.11.189:33805/ using document root <none> and password file <none>
20:08:24.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.840806 21640 fs_manager.cc:359] Metadata directory not provided
20:08:24.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.840919 21640 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:24.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.841233 21640 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.843781 21640 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/instance:
20:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773cf88af1d548c89309e0d6a065502c"
20:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.844517 21640 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal/instance:
20:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773cf88af1d548c89309e0d6a065502c"
20:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.849186 21640 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.005s	sys 0.001s
20:08:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.852582 21656 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.853812 21640 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.002s
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.853979 21640 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773cf88af1d548c89309e0d6a065502c"
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.854172 21640 fs_report.cc:383] FS layout report
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/data
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:24.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.865845 21640 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:24.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.866652 21640 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.866923 21640 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:24.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.872920 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } attempt: 4
20:08:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.874868 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.189:34501: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:24.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.886669 21640 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.189:34501
20:08:24.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.886675 21717 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.189:34501 every 8 connection(s)
20:08:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.888010 21640 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/info.pb
20:08:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.891428 21718 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.893847 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 21640
20:08:24.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.894047 21550 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal/instance
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.895300 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/logs
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data/info.pb
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.188:45949
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.188
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:08:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:08:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.898818 21718 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } has no permanent_uuid. Determining permanent_uuid...
20:08:24.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.912101 21718 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } has no permanent_uuid. Determining permanent_uuid...
20:08:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.920819 21718 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } has no permanent_uuid. Determining permanent_uuid...
20:08:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.921764 21657 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.923317 21718 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:24.972579 21718 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 1
20:08:24.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:24.974428 21718 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:25.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.042693 21718 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 2
20:08:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.044572 21718 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.135833 21718 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 3
20:08:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.137827 21718 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.174202 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } attempt: 5
20:08:25.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.180979 21637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } has no permanent_uuid. Determining permanent_uuid...
20:08:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.182811 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.204809 21720 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.205082 21720 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.205168 21720 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.211747 21720 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:08:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.211843 21720 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:08:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.211882 21720 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.211931 21720 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.211971 21720 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:08:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.212034 21720 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.212074 21720 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45863/jwks.json
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.220928 21720 master_runner.cc:386] Master server non-default flags:
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.188:45949
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data/info.pb
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.188
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/logs
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.222178 21720 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.223825 21720 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:25.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.234002 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 1
20:08:25.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.234241 21728 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:25.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.234791 21731 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:25.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.235023 21729 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:25.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.235828 21637 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:25.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.235860 21720 server_base.cc:1034] running on GCE node
20:08:25.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.236536 21720 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:25.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.237820 21720 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.239050 21720 hybrid_clock.cc:648] HybridClock initialized: now 1715026105238996 us; error 50 us; skew 500 ppm
20:08:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.239486 21720 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:25.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.242375 21720 webserver.cc:460] Webserver started at http://127.21.11.188:41445/ using document root <none> and password file <none>
20:08:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.243086 21720 fs_manager.cc:359] Metadata directory not provided
20:08:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.243193 21720 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.243489 21720 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.245855 21720 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data/instance:
20:08:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67c79f7042884b1ba605f75e40717256"
20:08:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:25 on dist-test-slave-zmnq"
20:08:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.246645 21720 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal/instance:
20:08:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67c79f7042884b1ba605f75e40717256"
20:08:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:25 on dist-test-slave-zmnq"
20:08:25.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.251892 21720 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.004s	sys 0.000s
20:08:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.255230 21737 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.256413 21720 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
20:08:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.256582 21720 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal
20:08:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67c79f7042884b1ba605f75e40717256"
20:08:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:25 on dist-test-slave-zmnq"
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.256798 21720 fs_report.cc:383] FS layout report
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data/data
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.272269 21720 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.273104 21720 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.273409 21720 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:25.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.281081 21718 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 4
20:08:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.283124 21718 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.293519 21720 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.188:45949
20:08:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.293540 21798 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.188:45949 every 8 connection(s)
20:08:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.294857 21720 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/data/info.pb
20:08:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.298123 21799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.302631 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 21720
20:08:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.302776 21550 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-2/wal/instance
20:08:25.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.304085 21637 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 2
20:08:25.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.303665 21799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } has no permanent_uuid. Determining permanent_uuid...
20:08:25.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.331688 21799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } has no permanent_uuid. Determining permanent_uuid...
20:08:25.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.337474 21637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Bootstrap starting.
20:08:25.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.341457 21637 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Neither blocks nor log segments found. Creating new log.
20:08:25.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.341426 21799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } has no permanent_uuid. Determining permanent_uuid...
20:08:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.343006 21637 log.cc:824] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Log is configured to *not* fsync() on all Append() calls
20:08:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.345441 21637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: No bootstrap required, opened a new log
20:08:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.351356 21799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: Bootstrap starting.
20:08:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.353593 21637 raft_consensus.cc:348] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.353936 21637 raft_consensus.cc:374] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.354040 21637 raft_consensus.cc:729] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 279092f506ba4af8b3d5dec9ce4ba0aa, State: Initialized, Role: FOLLOWER
20:08:25.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.354713 21637 consensus_queue.cc:260] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.355094 21799 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: Neither blocks nor log segments found. Creating new log.
20:08:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.355852 21808 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.356103 21808 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.356335 21799 log.cc:824] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: Log is configured to *not* fsync() on all Append() calls
20:08:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.357049 21637 sys_catalog.cc:564] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: configured and running, proceeding with master startup.
20:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.359133 21799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: No bootstrap required, opened a new log
20:08:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.369879 21799 raft_consensus.cc:348] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.370218 21799 raft_consensus.cc:374] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.370658 21799 raft_consensus.cc:729] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 67c79f7042884b1ba605f75e40717256, State: Initialized, Role: FOLLOWER
20:08:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.371394 21799 consensus_queue.cc:260] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.372179 21561 master_runner.cc:186] Error getting master registration for 127.21.11.189:34501: OK, instance_id { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" instance_seqno: 1715026104881642 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:08:25.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.387415 21561 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:08:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.390415 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.390548 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.391208 21822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.391580 21822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.392540 21799 sys_catalog.cc:564] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: configured and running, proceeding with master startup.
20:08:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.408900 21833 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.409020 21833 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:25.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.406946 21720 master_runner.cc:186] Error getting master registration for 127.21.11.189:34501: OK, instance_id { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" instance_seqno: 1715026104881642 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:08:25.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.415374 21720 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:08:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.440663 21808 raft_consensus.cc:482] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.440979 21808 raft_consensus.cc:504] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.442919 21808 leader_election.cc:288] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.443729 21764 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "67c79f7042884b1ba605f75e40717256" is_pre_election: true
20:08:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.444301 21764 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 279092f506ba4af8b3d5dec9ce4ba0aa in term 0.
20:08:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.444900 21576 leader_election.cc:302] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 67c79f7042884b1ba605f75e40717256; no voters: 
20:08:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.446244 21808 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Leader pre-election won for term 1
20:08:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.446405 21808 raft_consensus.cc:482] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:08:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.446502 21808 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 0 FOLLOWER]: Advancing to term 1
20:08:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.449626 21808 raft_consensus.cc:504] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.450693 21808 leader_election.cc:288] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 1 election: Requested vote from peers 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.451081 21764 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "67c79f7042884b1ba605f75e40717256"
20:08:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.451272 21764 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 0 FOLLOWER]: Advancing to term 1
20:08:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.454517 21764 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 279092f506ba4af8b3d5dec9ce4ba0aa in term 1.
20:08:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.455042 21576 leader_election.cc:302] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 67c79f7042884b1ba605f75e40717256; no voters: 
20:08:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.455442 21808 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 FOLLOWER]: Leader election won for term 1
20:08:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.456202 21808 raft_consensus.cc:686] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 LEADER]: Becoming Leader. State: Replica: 279092f506ba4af8b3d5dec9ce4ba0aa, State: Running, Role: LEADER
20:08:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.457074 21808 consensus_queue.cc:237] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.465194 21835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: New leader 279092f506ba4af8b3d5dec9ce4ba0aa. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.465544 21835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: LEADER
20:08:25.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:25.469997 21581 tablet.cc:2369] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:08:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.482954 21837 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:08:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.484992 21837 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:08:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.495730 21764 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Refusing update from remote peer 279092f506ba4af8b3d5dec9ce4ba0aa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:08:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.496770 21808 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.521446 21822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 279092f506ba4af8b3d5dec9ce4ba0aa. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.521775 21822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.523754 21838 mvcc.cc:205] Tried to move back new op lower bound from 7024746928096673792 to 7024746927961579520. Current Snapshot: MvccSnapshot[applied={T|T < 7024746928096673792}]
20:08:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.525002 21835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.525269 21835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: LEADER
20:08:25.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.525489 21839 mvcc.cc:205] Tried to move back new op lower bound from 7024746928096673792 to 7024746927961579520. Current Snapshot: MvccSnapshot[applied={T|T < 7024746928096673792}]
20:08:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.534109 21837 catalog_manager.cc:1347] Generated new cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.534233 21837 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:08:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.539760 21822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.539991 21822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.568079 21837 catalog_manager.cc:1370] Generated new certificate authority record
20:08:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.570351 21837 catalog_manager.cc:1504] Loading token signing keys...
20:08:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.582391 21718 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } attempt: 5
20:08:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.591027 21837 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Generated new TSK 0
20:08:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.592229 21837 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:08:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.594964 21718 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Bootstrap starting.
20:08:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.598546 21718 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Neither blocks nor log segments found. Creating new log.
20:08:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.599785 21718 log.cc:824] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Log is configured to *not* fsync() on all Append() calls
20:08:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.604115 21718 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: No bootstrap required, opened a new log
20:08:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.617838 21718 raft_consensus.cc:348] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.618351 21718 raft_consensus.cc:374] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.618557 21718 raft_consensus.cc:729] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 773cf88af1d548c89309e0d6a065502c, State: Initialized, Role: FOLLOWER
20:08:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.619630 21718 consensus_queue.cc:260] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.627907 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/logs
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data/info.pb
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.129:0
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.11.129
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.129
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:08:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.631342 21683 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "773cf88af1d548c89309e0d6a065502c" is_pre_election: true
20:08:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.631633 21681 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 0 FOLLOWER]: Advancing to term 1
20:08:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.632171 21682 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "773cf88af1d548c89309e0d6a065502c"
20:08:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.633553 21718 sys_catalog.cc:564] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: configured and running, proceeding with master startup.
20:08:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.634999 21844 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.635411 21844 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.638265 21681 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Refusing update from remote peer 279092f506ba4af8b3d5dec9ce4ba0aa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:08:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.639254 21835 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.679380 21844 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 279092f506ba4af8b3d5dec9ce4ba0aa. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.679831 21844 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.679359 21852 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.680265 21852 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: FOLLOWER
20:08:25.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.707201 21861 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:08:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.734638 21835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:25.734997 21835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: LEADER
20:08:26.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.053107 21846 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:26.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.053352 21846 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:26.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.053480 21846 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.061211 21846 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.061386 21846 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:08:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.061441 21846 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:08:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.061517 21846 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
20:08:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.061606 21846 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.11.129
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.069869 21846 tablet_server_runner.cc:78] Tablet server non-default flags:
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.129:0
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data/info.pb
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.129
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.11.129
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/logs
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.071132 21846 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.072816 21846 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.084053 21867 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.084410 21866 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.084980 21869 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.085714 21846 server_base.cc:1034] running on GCE node
20:08:26.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.086270 21846 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:26.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.087167 21846 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.088371 21846 hybrid_clock.cc:648] HybridClock initialized: now 1715026106088318 us; error 49 us; skew 500 ppm
20:08:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.088637 21846 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:26.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.090855 21846 webserver.cc:460] Webserver started at http://127.21.11.129:45027/ using document root <none> and password file <none>
20:08:26.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.091599 21846 fs_manager.cc:359] Metadata directory not provided
20:08:26.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.091733 21846 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:26.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.092098 21846 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.094748 21846 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data/instance:
20:08:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0669e5dad664465aacbbeeb3ed37a769"
20:08:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:26 on dist-test-slave-zmnq"
20:08:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.095536 21846 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal/instance:
20:08:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0669e5dad664465aacbbeeb3ed37a769"
20:08:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:26 on dist-test-slave-zmnq"
20:08:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.100597 21846 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
20:08:26.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.104100 21875 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.105257 21846 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.105383 21846 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0669e5dad664465aacbbeeb3ed37a769"
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:26 on dist-test-slave-zmnq"
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.105540 21846 fs_report.cc:383] FS layout report
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data/data
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.118734 21846 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.119431 21846 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.119702 21846 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.122673 21846 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:08:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.122776 21846 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:08:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.122906 21846 ts_tablet_manager.cc:590] Registered 0 tablets
20:08:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.122969 21846 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:08:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.182962 21846 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.129:37115
20:08:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.182992 21998 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.129:37115 every 8 connection(s)
20:08:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.184167 21846 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/data/info.pb
20:08:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.184515 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 21846
20:08:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.184779 21550 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-0/wal/instance
20:08:26.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.187966 22000 heartbeater.cc:347] Connected to a master server at 127.21.11.190:36769
20:08:26.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.188387 22000 heartbeater.cc:464] Registering TS with master...
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.190205 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/logs
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data/info.pb
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.130:0
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.11.130
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.130
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:08:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:08:26.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.195508 22000 heartbeater.cc:511] Master 127.21.11.190:36769 requested a full tablet report, sending...
20:08:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.205322 21591 ts_manager.cc:194] Registered new tserver with Master: 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115)
20:08:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.207684 21591 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.11.129:35571
20:08:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.210781 21999 heartbeater.cc:347] Connected to a master server at 127.21.11.188:45949
20:08:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.210924 21999 heartbeater.cc:464] Registering TS with master...
20:08:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.211436 22001 heartbeater.cc:347] Connected to a master server at 127.21.11.189:34501
20:08:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.211671 22001 heartbeater.cc:464] Registering TS with master...
20:08:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.212144 22001 heartbeater.cc:511] Master 127.21.11.189:34501 requested a full tablet report, sending...
20:08:26.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.212931 21999 heartbeater.cc:511] Master 127.21.11.188:45949 requested a full tablet report, sending...
20:08:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.213959 21673 ts_manager.cc:194] Registered new tserver with Master: 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115)
20:08:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.214888 21754 ts_manager.cc:194] Registered new tserver with Master: 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115)
20:08:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.411469 21833 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.411585 21833 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: loading cluster ID for follower catalog manager: success
20:08:26.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.415395 21833 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: acquiring CA information for follower catalog manager: success
20:08:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.417634 21833 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:08:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.520099 22004 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.520368 22004 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.520483 22004 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:26.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.527917 22004 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:26.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.528059 22004 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:08:26.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.528097 22004 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:08:26.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.528143 22004 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
20:08:26.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.528208 22004 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.11.130
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.537144 22004 tablet_server_runner.cc:78] Tablet server non-default flags:
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.130:0
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data/info.pb
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.130
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.11.130
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/logs
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.538615 22004 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:26.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.540350 22004 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.551221 22009 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.551754 22010 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:26.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.552435 22012 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:26.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.553587 22004 server_base.cc:1034] running on GCE node
20:08:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.554111 22004 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:26.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.554992 22004 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:26.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.556180 22004 hybrid_clock.cc:648] HybridClock initialized: now 1715026106556135 us; error 41 us; skew 500 ppm
20:08:26.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.556455 22004 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.558712 22004 webserver.cc:460] Webserver started at http://127.21.11.130:43829/ using document root <none> and password file <none>
20:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.559392 22004 fs_manager.cc:359] Metadata directory not provided
20:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.559516 22004 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.559757 22004 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.562201 22004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data/instance:
20:08:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c8f731f16954b8c864f9236254cdb02"
20:08:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:26 on dist-test-slave-zmnq"
20:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.562922 22004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal/instance:
20:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c8f731f16954b8c864f9236254cdb02"
20:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:26 on dist-test-slave-zmnq"
20:08:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.567253 22004 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.004s	sys 0.000s
20:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.570209 22018 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.571444 22004 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.571566 22004 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c8f731f16954b8c864f9236254cdb02"
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:26 on dist-test-slave-zmnq"
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.571713 22004 fs_report.cc:383] FS layout report
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data/data
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.592551 22004 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.593291 22004 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.593493 22004 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.596235 22004 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:08:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.596336 22004 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:08:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.596464 22004 ts_tablet_manager.cc:590] Registered 0 tablets
20:08:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.596532 22004 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:08:26.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.654179 22004 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.130:45751
20:08:26.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.654210 22139 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.130:45751 every 8 connection(s)
20:08:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.655344 22004 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/data/info.pb
20:08:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.655738 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 22004
20:08:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.655931 21550 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-1/wal/instance
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.661001 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/logs
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data/info.pb
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.131:0
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.11.131
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.131
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:08:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:08:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.673676 22141 heartbeater.cc:347] Connected to a master server at 127.21.11.190:36769
20:08:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.673950 22141 heartbeater.cc:464] Registering TS with master...
20:08:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.675026 22141 heartbeater.cc:511] Master 127.21.11.190:36769 requested a full tablet report, sending...
20:08:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.677294 21591 ts_manager.cc:194] Registered new tserver with Master: 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.678582 21591 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.11.130:52217
20:08:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.681455 22140 heartbeater.cc:347] Connected to a master server at 127.21.11.188:45949
20:08:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.681596 22140 heartbeater.cc:464] Registering TS with master...
20:08:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.682021 22140 heartbeater.cc:511] Master 127.21.11.188:45949 requested a full tablet report, sending...
20:08:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.679404 22142 heartbeater.cc:347] Connected to a master server at 127.21.11.189:34501
20:08:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.682456 22142 heartbeater.cc:464] Registering TS with master...
20:08:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.682783 22142 heartbeater.cc:511] Master 127.21.11.189:34501 requested a full tablet report, sending...
20:08:26.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.683205 21754 ts_manager.cc:194] Registered new tserver with Master: 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:26.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.683986 21673 ts_manager.cc:194] Registered new tserver with Master: 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:26.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.711853 21861 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:26.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.711997 21861 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: loading cluster ID for follower catalog manager: success
20:08:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.715826 21861 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: acquiring CA information for follower catalog manager: success
20:08:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.718184 21861 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:08:26.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.980850 22145 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:26.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.981151 22145 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:26.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.981298 22145 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:26.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.988298 22145 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:26.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.988467 22145 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:08:26.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.988520 22145 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:08:26.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.988584 22145 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
20:08:26.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:26.988672 22145 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.11.131
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.997313 22145 tablet_server_runner.cc:78] Tablet server non-default flags:
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.131:0
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data/info.pb
20:08:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.131
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.11.131
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/logs
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:26.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:26.998930 22145 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:27.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.000758 22145 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.013224 22152 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.013849 22155 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.014150 22153 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.014564 22145 server_base.cc:1034] running on GCE node
20:08:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.015118 22145 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:27.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.016033 22145 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:27.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.017235 22145 hybrid_clock.cc:648] HybridClock initialized: now 1715026107017177 us; error 52 us; skew 500 ppm
20:08:27.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.017560 22145 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:27.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.019781 22145 webserver.cc:460] Webserver started at http://127.21.11.131:34067/ using document root <none> and password file <none>
20:08:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.020562 22145 fs_manager.cc:359] Metadata directory not provided
20:08:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.020689 22145 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:27.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.021057 22145 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.024399 22145 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data/instance:
20:08:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b8380684688842afb1f56d7d8f29be21"
20:08:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:27 on dist-test-slave-zmnq"
20:08:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.025424 22145 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal/instance:
20:08:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b8380684688842afb1f56d7d8f29be21"
20:08:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:27 on dist-test-slave-zmnq"
20:08:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.032161 22145 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.002s	sys 0.003s
20:08:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.035766 22161 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.037165 22145 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.037421 22145 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b8380684688842afb1f56d7d8f29be21"
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:27 on dist-test-slave-zmnq"
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.037590 22145 fs_report.cc:383] FS layout report
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data/data
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:27.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:27.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:27.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:27.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.060534 22145 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.061345 22145 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.061630 22145 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.064785 22145 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.064890 22145 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.065016 22145 ts_tablet_manager.cc:590] Registered 0 tablets
20:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.065078 22145 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:08:27.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.115834 22145 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.131:36163
20:08:27.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.115850 22282 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.131:36163 every 8 connection(s)
20:08:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.117012 22145 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/data/info.pb
20:08:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.120209 22284 heartbeater.cc:347] Connected to a master server at 127.21.11.190:36769
20:08:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.120462 22284 heartbeater.cc:464] Registering TS with master...
20:08:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.121312 22284 heartbeater.cc:511] Master 127.21.11.190:36769 requested a full tablet report, sending...
20:08:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.122414 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 22145
20:08:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.122617 21550 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/ts-2/wal/instance
20:08:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.123792 21591 ts_manager.cc:194] Registered new tserver with Master: b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.124590 21591 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.11.131:42797
20:08:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.126801 22283 heartbeater.cc:347] Connected to a master server at 127.21.11.188:45949
20:08:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.127014 22283 heartbeater.cc:464] Registering TS with master...
20:08:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.127503 22283 heartbeater.cc:511] Master 127.21.11.188:45949 requested a full tablet report, sending...
20:08:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.128782 21753 ts_manager.cc:194] Registered new tserver with Master: b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.129019 22285 heartbeater.cc:347] Connected to a master server at 127.21.11.189:34501
20:08:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.129137 22285 heartbeater.cc:464] Registering TS with master...
20:08:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.129391 22285 heartbeater.cc:511] Master 127.21.11.189:34501 requested a full tablet report, sending...
20:08:27.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.130230 21673 ts_manager.cc:194] Registered new tserver with Master: b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.143294 21550 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:08:27.144 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:08:27.145 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

20:08:27.145 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.21.11.190"
      port: 36769
    }
    bound_http_address {
      host: "127.21.11.190"
      port: 36499
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.21.11.189"
      port: 34501
    }
    bound_http_address {
      host: "127.21.11.189"
      port: 33805
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.21.11.188"
      port: 45949
    }
    bound_http_address {
      host: "127.21.11.188"
      port: 41445
    }
  }
}

20:08:27.146 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

20:08:27.146 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.21.11.129"
      port: 37115
    }
    bound_http_address {
      host: "127.21.11.129"
      port: 45027
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.21.11.130"
      port: 45751
    }
    bound_http_address {
      host: "127.21.11.130"
      port: 43829
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.21.11.131"
      port: 36163
    }
    bound_http_address {
      host: "127.21.11.131"
      port: 34067
    }
  }
}

20:08:27.146 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:08:27.148 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
20:08:27.150 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.11.188:45949(127.21.11.188:45949)] connecting to peer
20:08:27.151 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:27.152 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.11.188:45949(127.21.11.188:45949)] Successfully connected to peer
20:08:27.153 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.11.189:34501(127.21.11.189:34501)] connecting to peer
20:08:27.154 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.11.190:36769(127.21.11.190:36769)] Successfully connected to peer
20:08:27.154 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.11.188
20:08:27.155 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.11.190
20:08:27.157 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.11.189:34501(127.21.11.189:34501)] Successfully connected to peer
20:08:27.167 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x656203b5, L:/127.0.0.1:53078 - R:/127.21.11.190:36769] using SASL/PLAIN
20:08:27.168 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x372b0856, L:/127.0.0.1:46972 - R:/127.21.11.188:45949] using SASL/PLAIN
20:08:27.168 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.11.189
20:08:27.170 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:08:27.171 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=22), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [20ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, deferred=null) after lookup
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.173128 21591 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53078:
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   hash_schema {
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
20:08:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     num_buckets: 2
20:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     seed: 0
20:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.175387 21591 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:08:27.182 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8bec42d2, L:/127.0.0.1:44556 - R:/127.21.11.189:34501] using SASL/PLAIN
20:08:27.205 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table d8ea12cfb2c84b01925f915dd71058b9
20:08:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.212905 22000 heartbeater.cc:503] Master 127.21.11.190:36769 was elected leader, sending a full tablet report...
20:08:27.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.245360 22074 tablet_service.cc:1467] Processing CreateTablet for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:08:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.246884 22072 tablet_service.cc:1467] Processing CreateTablet for tablet c8d53816408c47109c4d95dae94a247a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:08:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.247897 22072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8d53816408c47109c4d95dae94a247a. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.254601 22074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.259737 21933 tablet_service.cc:1467] Processing CreateTablet for tablet c8d53816408c47109c4d95dae94a247a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:08:27.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.259737 22217 tablet_service.cc:1467] Processing CreateTablet for tablet c8d53816408c47109c4d95dae94a247a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:08:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.261142 22217 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8d53816408c47109c4d95dae94a247a. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.261173 21934 tablet_service.cc:1467] Processing CreateTablet for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:08:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.261869 21934 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.262634 22218 tablet_service.cc:1467] Processing CreateTablet for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:08:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.263108 22218 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.263767 21933 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8d53816408c47109c4d95dae94a247a. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.271638 22308 tablet_bootstrap.cc:492] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: Bootstrap starting.
20:08:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.275424 22308 tablet_bootstrap.cc:654] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: Neither blocks nor log segments found. Creating new log.
20:08:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.276721 22308 log.cc:824] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: Log is configured to *not* fsync() on all Append() calls
20:08:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.282181 22310 tablet_bootstrap.cc:492] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: Bootstrap starting.
20:08:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.283351 22311 tablet_bootstrap.cc:492] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21: Bootstrap starting.
20:08:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.286270 22308 tablet_bootstrap.cc:492] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: No bootstrap required, opened a new log
20:08:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.286516 22310 tablet_bootstrap.cc:654] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: Neither blocks nor log segments found. Creating new log.
20:08:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.286690 22308 ts_tablet_manager.cc:1367] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: Time spent bootstrapping tablet: real 0.015s	user 0.010s	sys 0.002s
20:08:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.286804 22311 tablet_bootstrap.cc:654] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21: Neither blocks nor log segments found. Creating new log.
20:08:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.287767 22310 log.cc:824] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: Log is configured to *not* fsync() on all Append() calls
20:08:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.287976 22311 log.cc:824] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21: Log is configured to *not* fsync() on all Append() calls
20:08:27.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.297362 22310 tablet_bootstrap.cc:492] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: No bootstrap required, opened a new log
20:08:27.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.297824 22310 ts_tablet_manager.cc:1367] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: Time spent bootstrapping tablet: real 0.016s	user 0.001s	sys 0.012s
20:08:27.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.298739 22308 raft_consensus.cc:348] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.299367 22308 raft_consensus.cc:374] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.299607 22308 raft_consensus.cc:729] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c8f731f16954b8c864f9236254cdb02, State: Initialized, Role: FOLLOWER
20:08:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.300460 22308 consensus_queue.cc:260] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.309095 22311 tablet_bootstrap.cc:492] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21: No bootstrap required, opened a new log
20:08:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.309458 22311 ts_tablet_manager.cc:1367] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21: Time spent bootstrapping tablet: real 0.026s	user 0.006s	sys 0.006s
20:08:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.318038 22310 raft_consensus.cc:348] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.318497 22310 raft_consensus.cc:374] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.318596 22310 raft_consensus.cc:729] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0669e5dad664465aacbbeeb3ed37a769, State: Initialized, Role: FOLLOWER
20:08:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.319955 22310 consensus_queue.cc:260] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.320319 22308 ts_tablet_manager.cc:1398] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: Time spent starting tablet: real 0.033s	user 0.016s	sys 0.013s
20:08:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.320891 22308 tablet_bootstrap.cc:492] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02: Bootstrap starting.
20:08:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.322724 22308 tablet_bootstrap.cc:654] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02: Neither blocks nor log segments found. Creating new log.
20:08:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.320969 22141 heartbeater.cc:503] Master 127.21.11.190:36769 was elected leader, sending a full tablet report...
20:08:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.335542 22311 raft_consensus.cc:348] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.336355 22311 raft_consensus.cc:374] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.336920 22311 raft_consensus.cc:729] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b8380684688842afb1f56d7d8f29be21, State: Initialized, Role: FOLLOWER
20:08:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.338155 22311 consensus_queue.cc:260] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.341367 22310 ts_tablet_manager.cc:1398] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: Time spent starting tablet: real 0.043s	user 0.031s	sys 0.000s
20:08:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.342128 22308 tablet_bootstrap.cc:492] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02: No bootstrap required, opened a new log
20:08:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.342419 22308 ts_tablet_manager.cc:1367] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02: Time spent bootstrapping tablet: real 0.022s	user 0.015s	sys 0.005s
20:08:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.343467 22308 raft_consensus.cc:348] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.343791 22308 raft_consensus.cc:374] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.343976 22308 raft_consensus.cc:729] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c8f731f16954b8c864f9236254cdb02, State: Initialized, Role: FOLLOWER
20:08:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.344293 22308 consensus_queue.cc:260] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.344916 22308 ts_tablet_manager.cc:1398] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
20:08:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.342933 22311 ts_tablet_manager.cc:1398] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21: Time spent starting tablet: real 0.033s	user 0.016s	sys 0.001s
20:08:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.346238 22310 tablet_bootstrap.cc:492] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769: Bootstrap starting.
20:08:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.347733 22310 tablet_bootstrap.cc:654] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769: Neither blocks nor log segments found. Creating new log.
20:08:27.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.349573 22284 heartbeater.cc:503] Master 127.21.11.190:36769 was elected leader, sending a full tablet report...
20:08:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.350600 22311 tablet_bootstrap.cc:492] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21: Bootstrap starting.
20:08:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.352219 22311 tablet_bootstrap.cc:654] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21: Neither blocks nor log segments found. Creating new log.
20:08:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.356913 22311 tablet_bootstrap.cc:492] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21: No bootstrap required, opened a new log
20:08:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.357060 22311 ts_tablet_manager.cc:1367] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21: Time spent bootstrapping tablet: real 0.007s	user 0.001s	sys 0.003s
20:08:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.357329 22310 tablet_bootstrap.cc:492] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769: No bootstrap required, opened a new log
20:08:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.357473 22310 ts_tablet_manager.cc:1367] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769: Time spent bootstrapping tablet: real 0.014s	user 0.004s	sys 0.000s
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.357916 22311 raft_consensus.cc:348] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358112 22311 raft_consensus.cc:374] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358193 22311 raft_consensus.cc:729] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b8380684688842afb1f56d7d8f29be21, State: Initialized, Role: FOLLOWER
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358325 22310 raft_consensus.cc:348] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358506 22310 raft_consensus.cc:374] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358577 22310 raft_consensus.cc:729] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0669e5dad664465aacbbeeb3ed37a769, State: Initialized, Role: FOLLOWER
20:08:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358460 22311 consensus_queue.cc:260] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.358793 22310 consensus_queue.cc:260] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.359524 22310 ts_tablet_manager.cc:1398] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.002s
20:08:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.361744 22311 ts_tablet_manager.cc:1398] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21: Time spent starting tablet: real 0.005s	user 0.002s	sys 0.000s
20:08:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.365825 22314 raft_consensus.cc:482] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:27.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.366060 22314 raft_consensus.cc:504] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.367990 22314 leader_election.cc:288] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115), b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.369612 22286 tablet.cc:2369] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:08:27.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.375479 21954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8d53816408c47109c4d95dae94a247a" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0669e5dad664465aacbbeeb3ed37a769" is_pre_election: true
20:08:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.375934 21954 raft_consensus.cc:2454] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 0.
20:08:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.376523 22019 leader_election.cc:302] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0669e5dad664465aacbbeeb3ed37a769, 6c8f731f16954b8c864f9236254cdb02; no voters: 
20:08:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.376860 22314 raft_consensus.cc:2781] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:08:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.377032 22314 raft_consensus.cc:482] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:08:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.377135 22314 raft_consensus.cc:3037] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.377140 22238 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8d53816408c47109c4d95dae94a247a" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b8380684688842afb1f56d7d8f29be21" is_pre_election: true
20:08:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.377602 22238 raft_consensus.cc:2454] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 0.
20:08:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.380048 22314 raft_consensus.cc:504] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.380956 22322 raft_consensus.cc:482] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.381088 22322 raft_consensus.cc:504] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.381202 22238 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8d53816408c47109c4d95dae94a247a" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b8380684688842afb1f56d7d8f29be21"
20:08:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.381369 22238 raft_consensus.cc:3037] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.381723 22322 leader_election.cc:288] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115), b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.382443 22314 leader_election.cc:288] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 election: Requested vote from peers 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115), b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.382364 22237 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6cb0f0af93e4bcdbfac2ec6c3cfea9e" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b8380684688842afb1f56d7d8f29be21" is_pre_election: true
20:08:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.382688 22237 raft_consensus.cc:2454] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 0.
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.383255 21954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8d53816408c47109c4d95dae94a247a" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0669e5dad664465aacbbeeb3ed37a769"
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.383258 21953 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6cb0f0af93e4bcdbfac2ec6c3cfea9e" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0669e5dad664465aacbbeeb3ed37a769" is_pre_election: true
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.383507 21954 raft_consensus.cc:3037] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.383555 21953 raft_consensus.cc:2454] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 0.
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.384372 22238 raft_consensus.cc:2454] T c8d53816408c47109c4d95dae94a247a P b8380684688842afb1f56d7d8f29be21 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 1.
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.385347 21954 raft_consensus.cc:2454] T c8d53816408c47109c4d95dae94a247a P 0669e5dad664465aacbbeeb3ed37a769 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 1.
20:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.385643 22019 leader_election.cc:302] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0669e5dad664465aacbbeeb3ed37a769, 6c8f731f16954b8c864f9236254cdb02; no voters: 
20:08:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.385903 22021 leader_election.cc:302] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6c8f731f16954b8c864f9236254cdb02, b8380684688842afb1f56d7d8f29be21; no voters: 
20:08:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.386092 22314 raft_consensus.cc:2781] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:08:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.386171 22314 raft_consensus.cc:482] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:08:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.386265 22314 raft_consensus.cc:3037] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.387930 22314 raft_consensus.cc:504] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.388643 22322 raft_consensus.cc:2781] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 1 FOLLOWER]: Leader election won for term 1
20:08:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.388834 21954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6cb0f0af93e4bcdbfac2ec6c3cfea9e" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0669e5dad664465aacbbeeb3ed37a769"
20:08:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.388880 22238 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6cb0f0af93e4bcdbfac2ec6c3cfea9e" candidate_uuid: "6c8f731f16954b8c864f9236254cdb02" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b8380684688842afb1f56d7d8f29be21"
20:08:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.389058 21954 raft_consensus.cc:3037] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.389086 22238 raft_consensus.cc:3037] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.392534 22238 raft_consensus.cc:2454] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P b8380684688842afb1f56d7d8f29be21 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 1.
20:08:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.388490 22314 leader_election.cc:288] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 election: Requested vote from peers 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115), b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.393535 21954 raft_consensus.cc:2454] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c8f731f16954b8c864f9236254cdb02 in term 1.
20:08:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.393946 22019 leader_election.cc:302] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0669e5dad664465aacbbeeb3ed37a769, 6c8f731f16954b8c864f9236254cdb02; no voters: 
20:08:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.394717 22314 raft_consensus.cc:2781] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 1 FOLLOWER]: Leader election won for term 1
20:08:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.394999 22314 raft_consensus.cc:686] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [term 1 LEADER]: Becoming Leader. State: Replica: 6c8f731f16954b8c864f9236254cdb02, State: Running, Role: LEADER
20:08:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.395051 22322 raft_consensus.cc:686] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [term 1 LEADER]: Becoming Leader. State: Replica: 6c8f731f16954b8c864f9236254cdb02, State: Running, Role: LEADER
20:08:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.395612 22322 consensus_queue.cc:237] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.395622 22314 consensus_queue.cc:237] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.400641 21590 catalog_manager.cc:5484] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130). New cstate: current_term: 1 leader_uuid: "6c8f731f16954b8c864f9236254cdb02" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } health_report { overall_health: UNKNOWN } } }
20:08:27.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.408984 21590 catalog_manager.cc:5484] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130). New cstate: current_term: 1 leader_uuid: "6c8f731f16954b8c864f9236254cdb02" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } health_report { overall_health: UNKNOWN } } }
20:08:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.430356 22143 tablet.cc:2369] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:08:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.464293 22002 tablet.cc:2369] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 0669e5dad664465aacbbeeb3ed37a769: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.470708 21590 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53094:
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:08:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:08:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.471760 21590 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:08:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.483731 22217 tablet_service.cc:1467] Processing CreateTablet for tablet 089f8012fcca4b2b97c016dd5e10d3df (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:08:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.483486 21933 tablet_service.cc:1467] Processing CreateTablet for tablet 089f8012fcca4b2b97c016dd5e10d3df (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:08:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.483975 22072 tablet_service.cc:1467] Processing CreateTablet for tablet 089f8012fcca4b2b97c016dd5e10d3df (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:08:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.484519 21933 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 089f8012fcca4b2b97c016dd5e10d3df. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.484521 22217 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 089f8012fcca4b2b97c016dd5e10d3df. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.484746 22072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 089f8012fcca4b2b97c016dd5e10d3df. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:27.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.492013 22310 tablet_bootstrap.cc:492] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: Bootstrap starting.
20:08:27.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.492556 22311 tablet_bootstrap.cc:492] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: Bootstrap starting.
20:08:27.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.493315 22308 tablet_bootstrap.cc:492] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: Bootstrap starting.
20:08:27.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.493683 22310 tablet_bootstrap.cc:654] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: Neither blocks nor log segments found. Creating new log.
20:08:27.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.494071 22311 tablet_bootstrap.cc:654] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: Neither blocks nor log segments found. Creating new log.
20:08:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.495136 22308 tablet_bootstrap.cc:654] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: Neither blocks nor log segments found. Creating new log.
20:08:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.495919 22310 tablet_bootstrap.cc:492] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: No bootstrap required, opened a new log
20:08:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.496060 22310 ts_tablet_manager.cc:1367] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: Time spent bootstrapping tablet: real 0.004s	user 0.000s	sys 0.003s
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.496383 22311 tablet_bootstrap.cc:492] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: No bootstrap required, opened a new log
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.496542 22311 ts_tablet_manager.cc:1367] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: Time spent bootstrapping tablet: real 0.004s	user 0.003s	sys 0.000s
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.496848 22310 raft_consensus.cc:348] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497079 22310 raft_consensus.cc:374] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497155 22310 raft_consensus.cc:729] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0669e5dad664465aacbbeeb3ed37a769, State: Initialized, Role: FOLLOWER
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497185 22308 tablet_bootstrap.cc:492] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: No bootstrap required, opened a new log
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497330 22308 ts_tablet_manager.cc:1367] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: Time spent bootstrapping tablet: real 0.004s	user 0.003s	sys 0.000s
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497460 22310 consensus_queue.cc:260] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497253 22311 raft_consensus.cc:348] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497814 22311 raft_consensus.cc:374] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.497865 22311 raft_consensus.cc:729] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b8380684688842afb1f56d7d8f29be21, State: Initialized, Role: FOLLOWER
20:08:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498039 22308 raft_consensus.cc:348] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498111 22311 consensus_queue.cc:260] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498278 22308 raft_consensus.cc:374] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498428 22308 raft_consensus.cc:729] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c8f731f16954b8c864f9236254cdb02, State: Initialized, Role: FOLLOWER
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498421 22315 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498780 22315 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.498682 22308 consensus_queue.cc:260] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.499264 22308 ts_tablet_manager.cc:1398] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.499171 22314 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.499433 22314 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.499833 22310 ts_tablet_manager.cc:1398] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: Time spent starting tablet: real 0.004s	user 0.000s	sys 0.002s
20:08:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.503252 22317 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.503645 22317 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.506139 22311 ts_tablet_manager.cc:1398] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: Time spent starting tablet: real 0.010s	user 0.001s	sys 0.000s
20:08:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.562695 22317 raft_consensus.cc:482] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:27.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.562903 22317 raft_consensus.cc:504] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.564419 22317 leader_election.cc:288] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115), 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.571097 21954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "089f8012fcca4b2b97c016dd5e10d3df" candidate_uuid: "b8380684688842afb1f56d7d8f29be21" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0669e5dad664465aacbbeeb3ed37a769" is_pre_election: true
20:08:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.571288 21954 raft_consensus.cc:2454] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b8380684688842afb1f56d7d8f29be21 in term 0.
20:08:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.571218 22095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "089f8012fcca4b2b97c016dd5e10d3df" candidate_uuid: "b8380684688842afb1f56d7d8f29be21" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6c8f731f16954b8c864f9236254cdb02" is_pre_election: true
20:08:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.571606 22095 raft_consensus.cc:2454] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b8380684688842afb1f56d7d8f29be21 in term 0.
20:08:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.571656 22162 leader_election.cc:302] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0669e5dad664465aacbbeeb3ed37a769, b8380684688842afb1f56d7d8f29be21; no voters: 
20:08:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.572027 22317 raft_consensus.cc:2781] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:08:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.572124 22317 raft_consensus.cc:482] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:08:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.572190 22317 raft_consensus.cc:3037] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.573946 22317 raft_consensus.cc:504] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.574540 22317 leader_election.cc:288] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [CANDIDATE]: Term 1 election: Requested vote from peers 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115), 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.574916 21954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "089f8012fcca4b2b97c016dd5e10d3df" candidate_uuid: "b8380684688842afb1f56d7d8f29be21" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0669e5dad664465aacbbeeb3ed37a769"
20:08:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.574990 22095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "089f8012fcca4b2b97c016dd5e10d3df" candidate_uuid: "b8380684688842afb1f56d7d8f29be21" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6c8f731f16954b8c864f9236254cdb02"
20:08:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.575083 21954 raft_consensus.cc:3037] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.575156 22095 raft_consensus.cc:3037] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 0 FOLLOWER]: Advancing to term 1
20:08:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.576690 21954 raft_consensus.cc:2454] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b8380684688842afb1f56d7d8f29be21 in term 1.
20:08:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.577024 22162 leader_election.cc:302] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0669e5dad664465aacbbeeb3ed37a769, b8380684688842afb1f56d7d8f29be21; no voters: 
20:08:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.577073 22095 raft_consensus.cc:2454] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b8380684688842afb1f56d7d8f29be21 in term 1.
20:08:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.577255 22317 raft_consensus.cc:2781] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 1 FOLLOWER]: Leader election won for term 1
20:08:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.577618 22317 raft_consensus.cc:686] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [term 1 LEADER]: Becoming Leader. State: Replica: b8380684688842afb1f56d7d8f29be21, State: Running, Role: LEADER
20:08:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.578111 22317 consensus_queue.cc:237] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } }
20:08:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.580736 22337 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: TxnStatusTablet state changed. Reason: New leader b8380684688842afb1f56d7d8f29be21. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.581054 22337 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: This TxnStatusTablet replica's current role is: LEADER
20:08:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.581657 22339 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
20:08:27.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.581959 22339 txn_status_manager.cc:929] Loading transaction status metadata into memory...
20:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.583349 21591 catalog_manager.cc:5484] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 reported cstate change: term changed from 0 to 1, leader changed from <none> to b8380684688842afb1f56d7d8f29be21 (127.21.11.131). New cstate: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } health_report { overall_health: HEALTHY } } }
20:08:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.614244 21954 raft_consensus.cc:1261] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769 [term 1 FOLLOWER]: Refusing update from remote peer b8380684688842afb1f56d7d8f29be21: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:08:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.614362 22095 raft_consensus.cc:1261] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02 [term 1 FOLLOWER]: Refusing update from remote peer b8380684688842afb1f56d7d8f29be21: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:08:27.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.615178 22337 consensus_queue.cc:1035] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:27.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.615437 22317 consensus_queue.cc:1035] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.621771 22314 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: TxnStatusTablet state changed. Reason: New leader b8380684688842afb1f56d7d8f29be21. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.622077 22314 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.626793 22342 mvcc.cc:205] Tried to move back new op lower bound from 7024746936779665408 to 7024746936643747840. Current Snapshot: MvccSnapshot[applied={T|T < 7024746936779665408}]
20:08:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.632686 22317 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.633036 22317 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: This TxnStatusTablet replica's current role is: LEADER
20:08:27.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.635550 22314 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.635762 22314 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P 6c8f731f16954b8c864f9236254cdb02: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.640035 22315 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: TxnStatusTablet state changed. Reason: New leader b8380684688842afb1f56d7d8f29be21. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.640300 22315 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.642 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
20:08:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.643482 22337 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.643723 22337 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21: This TxnStatusTablet replica's current role is: LEADER
20:08:27.649 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table d8ea12cfb2c84b01925f915dd71058b9
20:08:27.650 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.11.190:36769
20:08:27.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

20:08:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.654415 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 21561
20:08:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.655141 22344 mvcc.cc:205] Tried to move back new op lower bound from 7024746936779665408 to 7024746936643747840. Current Snapshot: MvccSnapshot[applied={T|T < 7024746936779665408}]
20:08:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.655689 21632 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:27.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.665182 22315 tablet_replica.cc:439] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b8380684688842afb1f56d7d8f29be21" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 } } peers { permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" member_type: VOTER last_known_addr { host: "127.21.11.130" port: 45751 } } peers { permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 } } }
20:08:27.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.665403 22315 tablet_replica.cc:442] T 089f8012fcca4b2b97c016dd5e10d3df P 0669e5dad664465aacbbeeb3ed37a769: This TxnStatusTablet replica's current role is: FOLLOWER
20:08:27.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.787240 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 21561
20:08:27.810 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.11.190:36769(127.21.11.190:36769)] handling channelInactive
20:08:27.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.810410 21660 connection.cc:535] server connection from 127.0.0.1:44542 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:27.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state READY due to: connection closed
20:08:27.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:08:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.837872 22314 consensus_queue.cc:1035] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.843871 22000 heartbeater.cc:644] Failed to heartbeat to 127.21.11.190:36769 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:27.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.846172 22141 heartbeater.cc:644] Failed to heartbeat to 127.21.11.190:36769 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.846570 22323 consensus_queue.cc:1035] T c8d53816408c47109c4d95dae94a247a P 6c8f731f16954b8c864f9236254cdb02 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:27.851600 22284 heartbeater.cc:644] Failed to heartbeat to 127.21.11.190:36769 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:27.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.869593 22323 consensus_queue.cc:1035] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" member_type: VOTER last_known_addr { host: "127.21.11.129" port: 37115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:27.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:27.875674 22314 consensus_queue.cc:1035] T b6cb0f0af93e4bcdbfac2ec6c3cfea9e P 6c8f731f16954b8c864f9236254cdb02 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b8380684688842afb1f56d7d8f29be21" member_type: VOTER last_known_addr { host: "127.21.11.131" port: 36163 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:08:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.088078 22357 raft_consensus.cc:482] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 279092f506ba4af8b3d5dec9ce4ba0aa)
20:08:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.088296 22357 raft_consensus.cc:504] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.089164 22358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 279092f506ba4af8b3d5dec9ce4ba0aa)
20:08:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.089341 22358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.089658 22357 leader_election.cc:288] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501)
20:08:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.090471 21738 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.090601 22358 leader_election.cc:288] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.090759 21682 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67c79f7042884b1ba605f75e40717256" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "773cf88af1d548c89309e0d6a065502c" is_pre_election: true
20:08:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.091035 21682 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 67c79f7042884b1ba605f75e40717256 in term 1.
20:08:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.092033 21763 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "773cf88af1d548c89309e0d6a065502c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "67c79f7042884b1ba605f75e40717256" is_pre_election: true
20:08:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.092339 21763 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 773cf88af1d548c89309e0d6a065502c in term 1.
20:08:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.092278 21738 leader_election.cc:302] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 67c79f7042884b1ba605f75e40717256, 773cf88af1d548c89309e0d6a065502c; no voters: 
20:08:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.092700 21738 leader_election.cc:334] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.092859 22357 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:08:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.092983 22357 raft_consensus.cc:482] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 279092f506ba4af8b3d5dec9ce4ba0aa)
20:08:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.093139 22357 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 1 FOLLOWER]: Advancing to term 2
20:08:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.093484 21657 leader_election.cc:302] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 67c79f7042884b1ba605f75e40717256, 773cf88af1d548c89309e0d6a065502c; no voters: 
20:08:28.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.093884 22358 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Leader pre-election won for term 2
20:08:28.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.093891 21657 leader_election.cc:334] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.094003 22358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Starting leader election (detected failure of leader 279092f506ba4af8b3d5dec9ce4ba0aa)
20:08:28.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.094147 22358 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 1 FOLLOWER]: Advancing to term 2
20:08:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.095366 22357 raft_consensus.cc:504] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.095813 22357 leader_election.cc:288] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 2 election: Requested vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501)
20:08:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.096139 22358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.096483 21682 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67c79f7042884b1ba605f75e40717256" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "773cf88af1d548c89309e0d6a065502c"
20:08:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.096766 21682 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 67c79f7042884b1ba605f75e40717256 in current term 2: Already voted for candidate 773cf88af1d548c89309e0d6a065502c in this term.
20:08:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.096890 22358 leader_election.cc:288] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 2 election: Requested vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.096993 21738 leader_election.cc:334] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.097555 21763 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "773cf88af1d548c89309e0d6a065502c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "67c79f7042884b1ba605f75e40717256"
20:08:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.097633 21738 leader_election.cc:302] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 67c79f7042884b1ba605f75e40717256; no voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 773cf88af1d548c89309e0d6a065502c
20:08:28.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.097846 21763 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 773cf88af1d548c89309e0d6a065502c in current term 2: Already voted for candidate 67c79f7042884b1ba605f75e40717256 in this term.
20:08:28.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.097999 22357 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
20:08:28.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.098441 21657 leader_election.cc:334] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.098559 21657 leader_election.cc:302] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 773cf88af1d548c89309e0d6a065502c; no voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 67c79f7042884b1ba605f75e40717256
20:08:28.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.098788 22358 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
20:08:28.150 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.151 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.151 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.152 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.21.11.190:36769(127.21.11.190:36769) for tablet Kudu Master: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.170 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.190 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.210 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.230 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.250 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.310 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.312 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] sending RPC to server master-127.21.11.190:36769, [1ms] received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [83ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, deferred=Deferred@1848340793(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@52049ee3 -> (continuation of Deferred@290998677 after retry RPC after error@1151868203) -> (continuation of Deferred@1899961918 after retry RPC after error@121271676) -> (continuation of Deferred@1417511151 after retry RPC after error@1354272270) -> (continuation of Deferred@1501541577 after retry RPC after error@221948687) -> (continuation of Deferred@905455807 after retry RPC after error@556009712), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@5d939e1 -> (continuation of Deferred@290998677 after retry RPC after error@1151868203) -> (continuation of Deferred@1899961918 after retry RPC after error@121271676) -> (continuation of Deferred@1417511151 after retry RPC after error@1354272270) -> (continuation of Deferred@1501541577 after retry RPC after error@221948687) -> (continuation of Deferred@905455807 after retry RPC after error@556009712)))
20:08:28.313 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] sending RPC to server master-127.21.11.190:36769, [1ms] received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [83ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, deferred=Deferred@1848340793(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@52049ee3 -> (continuation of Deferred@290998677 after retry RPC after error@1151868203) -> (continuation of Deferred@1899961918 after retry RPC after error@121271676) -> (continuation of Deferred@1417511151 after retry RPC after error@1354272270) -> (continuation of Deferred@1501541577 after retry RPC after error@221948687) -> (continuation of Deferred@905455807 after retry RPC after error@556009712), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@5d939e1 -> (continuation of Deferred@290998677 after retry RPC after error@1151868203) -> (continuation of Deferred@1899961918 after retry RPC after error@121271676) -> (continuation of Deferred@1417511151 after retry RPC after error@1354272270) -> (continuation of Deferred@1501541577 after retry RPC after error@221948687) -> (continuation of Deferred@905455807 after retry RPC after error@556009712)))
20:08:28.330 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.350 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.351 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.351 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.370 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.450 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.450 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.457103 22358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.457314 22358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.457912 22358 leader_election.cc:288] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:28.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.458714 21763 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "773cf88af1d548c89309e0d6a065502c" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "67c79f7042884b1ba605f75e40717256" is_pre_election: true
20:08:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.458948 21763 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 773cf88af1d548c89309e0d6a065502c in term 2.
20:08:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.459189 21657 leader_election.cc:334] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.459419 21657 leader_election.cc:302] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67c79f7042884b1ba605f75e40717256, 773cf88af1d548c89309e0d6a065502c; no voters: 279092f506ba4af8b3d5dec9ce4ba0aa
20:08:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.459707 22358 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Leader pre-election won for term 3
20:08:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.459806 22358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:08:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.459900 22358 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 2 FOLLOWER]: Advancing to term 3
20:08:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.461764 22358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.462368 22358 leader_election.cc:288] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 3 election: Requested vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.462998 21763 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "773cf88af1d548c89309e0d6a065502c" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "67c79f7042884b1ba605f75e40717256"
20:08:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.463214 21763 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 2 FOLLOWER]: Advancing to term 3
20:08:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.463480 21657 leader_election.cc:334] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111)
20:08:28.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.464979 21763 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 773cf88af1d548c89309e0d6a065502c in term 3.
20:08:28.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.465348 21657 leader_election.cc:302] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67c79f7042884b1ba605f75e40717256, 773cf88af1d548c89309e0d6a065502c; no voters: 279092f506ba4af8b3d5dec9ce4ba0aa
20:08:28.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.465592 22358 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 3 FOLLOWER]: Leader election won for term 3
20:08:28.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.465965 22358 raft_consensus.cc:686] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 3 LEADER]: Becoming Leader. State: Replica: 773cf88af1d548c89309e0d6a065502c, State: Running, Role: LEADER
20:08:28.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.466367 22358 consensus_queue.cc:237] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:28.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.467613 22363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 773cf88af1d548c89309e0d6a065502c. Latest consensus state: current_term: 3 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:28.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.467844 22363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: LEADER
20:08:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.468457 22364 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:08:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.470664 22364 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339]
20:08:28.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.471117 22364 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]
20:08:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.472584 22364 tablet_loader.cc:96] loaded metadata for tablet 089f8012fcca4b2b97c016dd5e10d3df (table kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339])
20:08:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.472985 22364 tablet_loader.cc:96] loaded metadata for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e (table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9])
20:08:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.473276 22364 tablet_loader.cc:96] loaded metadata for tablet c8d53816408c47109c4d95dae94a247a (table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9])
20:08:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.473479 22364 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:08:28.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.474237 22364 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:28.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.474342 22364 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:08:28.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.475536 22364 catalog_manager.cc:1504] Loading token signing keys...
20:08:28.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.476399 22364 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Loaded TSK: 0
20:08:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.477142 22364 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:08:28.490 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:28.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.190:36769: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769
20:08:28.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:08:28.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [3ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [162ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: Network error: Failed to connect to peer master-127.21.11.190:36769(127.21.11.190:36769): Connection refused: /127.21.11.190:36769, [162ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, deferred=Deferred@774737258(state=PENDING, result=null, callback=(continuation of Deferred@1962352864 after retry RPC after error@2013036619) -> (continuation of Deferred@252912686 after retry RPC after error@1301786235) -> (continuation of Deferred@2782985 after retry RPC after error@323441055) -> (continuation of Deferred@90383757 after retry RPC after error@1970775852) -> (continuation of Deferred@649462964 after retry RPC after error@265577044) -> (continuation of Deferred@319254894 after retry RPC after error@1535245029), errback=(continuation of Deferred@1962352864 after retry RPC after error@2013036619) -> (continuation of Deferred@252912686 after retry RPC after error@1301786235) -> (continuation of Deferred@2782985 after retry RPC after error@323441055) -> (continuation of Deferred@90383757 after retry RPC after error@1970775852) -> (continuation of Deferred@649462964 after retry RPC after error@265577044) -> (continuation of Deferred@319254894 after retry RPC after error@1535245029))) after lookup
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.535125 21673 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44568:
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.540771 21673 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:08:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:28.572331 21657 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Couldn't send request to peer 279092f506ba4af8b3d5dec9ce4ba0aa. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:08:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.587916 21763 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Refusing update from remote peer 773cf88af1d548c89309e0d6a065502c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
20:08:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.588587 22363 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [LEADER]: Connected to new peer: Peer: permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
20:08:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.589865 22357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 773cf88af1d548c89309e0d6a065502c. Latest consensus state: current_term: 3 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.590142 22357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: FOLLOWER
20:08:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.591310 22358 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.591490 22358 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: LEADER
20:08:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.591787 22357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:28.592020 22357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: FOLLOWER
20:08:29.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:29.026711 21657 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Couldn't send request to peer 279092f506ba4af8b3d5dec9ce4ba0aa. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:08:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:29.544559 21657 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Couldn't send request to peer 279092f506ba4af8b3d5dec9ce4ba0aa. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:08:29.816 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
20:08:29.816 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet c8d53816408c47109c4d95dae94a247a for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
20:08:29.816 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=b6cb0f0af93e4bcdbfac2ec6c3cfea9e@[0669e5dad664465aacbbeeb3ed37a769(127.21.11.129:37115),6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)[L],b8380684688842afb1f56d7d8f29be21(127.21.11.131:36163)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=c8d53816408c47109c4d95dae94a247a@[0669e5dad664465aacbbeeb3ed37a769(127.21.11.129:37115),6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)[L],b8380684688842afb1f56d7d8f29be21(127.21.11.131:36163)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
20:08:29.816 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.11.189:34501, [1ms] Sub RPC GetTableLocations: received response from server master-127.21.11.189:34501: OK, deferred=null) after lookup
20:08:29.817 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)] connecting to peer
20:08:29.817 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)] Successfully connected to peer
20:08:29.831 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc66a6471, L:/127.0.0.1:49810 - R:/127.21.11.130:45751] using TOKEN/null
20:08:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:29.835187 22061 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.21.11.190:36769)
20:08:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:29.882443 22001 heartbeater.cc:503] Master 127.21.11.189:34501 was elected leader, sending a full tablet report...
20:08:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:29.884301 22285 heartbeater.cc:503] Master 127.21.11.189:34501 was elected leader, sending a full tablet report...
20:08:29.895 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.11.190:36769
20:08:29.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:29.896500 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/logs
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/info.pb
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.190:36769
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.190
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36499
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:08:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:08:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.012326 21657 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111) [suppressed 22 similar messages]
20:08:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.137393 21657 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769): Couldn't send request to peer 279092f506ba4af8b3d5dec9ce4ba0aa. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.190:36769: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
20:08:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.205344 22404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.205624 22404 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.205734 22404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.212525 22404 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.212647 22404 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.212703 22404 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.212790 22404 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.212852 22404 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:08:30.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.212934 22404 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:30.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.213003 22404 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45863/jwks.json
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.221122 22404 master_runner.cc:386] Master server non-default flags:
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.190:36769
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/info.pb
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.190
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36499
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/logs
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.222723 22404 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.224335 22404 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.234748 22409 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.235046 22412 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.235719 22410 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.236506 22404 server_base.cc:1034] running on GCE node
20:08:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.237215 22404 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.238482 22404 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.239673 22404 hybrid_clock.cc:648] HybridClock initialized: now 1715026110239627 us; error 39 us; skew 500 ppm
20:08:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.239980 22404 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:30.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.242302 22404 webserver.cc:460] Webserver started at http://127.21.11.190:36499/ using document root <none> and password file <none>
20:08:30.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.243005 22404 fs_manager.cc:359] Metadata directory not provided
20:08:30.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.243108 22404 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:30.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.247781 22404 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.002s	sys 0.003s
20:08:30.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.250392 22418 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.251513 22404 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.251652 22404 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "279092f506ba4af8b3d5dec9ce4ba0aa"
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.252427 22404 fs_report.cc:383] FS layout report
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/wal
20:08:30.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/data
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.278827 22404 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.279655 22404 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.279898 22404 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.301525 22404 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.190:36769
20:08:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.301543 22479 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.190:36769 every 8 connection(s)
20:08:30.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.303421 22404 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-0/data/info.pb
20:08:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.306310 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 22404
20:08:30.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:08:30.306 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.11.189:34501
20:08:30.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

20:08:30.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.307138 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 21640
20:08:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.308179 21713 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:30.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.310956 22480 sys_catalog.cc:263] Verifying existing consensus state
20:08:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.313995 22480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Bootstrap starting.
20:08:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.328498 22480 log.cc:824] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Log is configured to *not* fsync() on all Append() calls
20:08:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.340672 22480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:08:30.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.341153 22480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: Bootstrap complete.
20:08:30.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.348655 22480 raft_consensus.cc:348] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:30.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.349378 22480 raft_consensus.cc:729] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 279092f506ba4af8b3d5dec9ce4ba0aa, State: Initialized, Role: FOLLOWER
20:08:30.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.350019 22480 consensus_queue.cc:260] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.350863 22484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.351184 22484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.351713 22480 sys_catalog.cc:564] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: configured and running, proceeding with master startup.
20:08:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.362356 22495 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:30.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.362939 22495 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: loading cluster ID for follower catalog manager: success
20:08:30.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.369896 22495 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: acquiring CA information for follower catalog manager: success
20:08:30.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.372407 22495 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:08:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.392062 22142 heartbeater.cc:503] Master 127.21.11.189:34501 was elected leader, sending a full tablet report...
20:08:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.398419 22141 heartbeater.cc:347] Connected to a master server at 127.21.11.190:36769
20:08:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.402009 22435 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" instance_seqno: 1715026106647160) as {username='slave'} at 127.21.11.130:39173; Asking this server to re-register.
20:08:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.402877 22141 heartbeater.cc:464] Registering TS with master...
20:08:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.404506 22435 ts_manager.cc:194] Registered new tserver with Master: 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.437477 21720 master_runner.cc:232] Existing masters have differing terms: 1 vs 3
20:08:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.437798 21720 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:08:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.437963 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 21640
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.464046 22497 negotiation.cc:337] Failed RPC negotiation. Trace:
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:30.384320 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.21.11.189:34501 (local address 127.0.0.1:44582)
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:30.384490 (+   170us) negotiation.cc:107] Waiting for socket to connect
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:30.384508 (+    18us) client_negotiation.cc:174] Beginning negotiation
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:30.384654 (+   146us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:30.462810 (+ 78156us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":61}
20:08:30.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:08:30.466 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.11.189:34501(127.21.11.189:34501)] handling channelInactive
20:08:30.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.189:34501(127.21.11.189:34501)] cleaning up while in state READY due to: connection closed
20:08:30.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.465000 22404 master_runner.cc:186] Error getting master registration for 127.21.11.189:34501: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107), 
20:08:30.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.470182 22404 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:08:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.585000 22503 raft_consensus.cc:482] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 773cf88af1d548c89309e0d6a065502c)
20:08:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.585216 22503 raft_consensus.cc:504] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.586033 22503 leader_election.cc:288] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501)
20:08:30.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.586865 22445 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67c79f7042884b1ba605f75e40717256" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" is_pre_election: true
20:08:30.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.587448 22445 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 67c79f7042884b1ba605f75e40717256 in term 1.
20:08:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.587987 21738 leader_election.cc:302] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 67c79f7042884b1ba605f75e40717256; no voters: 
20:08:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.588284 22503 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Leader pre-election won for term 4
20:08:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.588411 22503 raft_consensus.cc:482] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 773cf88af1d548c89309e0d6a065502c)
20:08:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.588544 22503 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 3 FOLLOWER]: Advancing to term 4
20:08:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.588819 21738 leader_election.cc:334] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.590523 22503 raft_consensus.cc:504] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.591041 22503 leader_election.cc:288] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 4 election: Requested vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501)
20:08:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.591557 22445 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67c79f7042884b1ba605f75e40717256" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa"
20:08:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.591760 22445 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 1 FOLLOWER]: Advancing to term 4
20:08:30.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.592479 21738 leader_election.cc:334] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111)
20:08:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.596251 22445 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 67c79f7042884b1ba605f75e40717256 in term 4.
20:08:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.596632 21738 leader_election.cc:302] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 67c79f7042884b1ba605f75e40717256; no voters: 773cf88af1d548c89309e0d6a065502c
20:08:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.596974 22503 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 4 FOLLOWER]: Leader election won for term 4
20:08:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.597394 22503 raft_consensus.cc:686] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [term 4 LEADER]: Becoming Leader. State: Replica: 67c79f7042884b1ba605f75e40717256, State: Running, Role: LEADER
20:08:30.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.597837 22503 consensus_queue.cc:237] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.599412 22505 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 67c79f7042884b1ba605f75e40717256. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.599807 22505 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: LEADER
20:08:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.600414 22507 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:08:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.602537 22507 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339]
20:08:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.603068 22507 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]
20:08:30.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.604552 22507 tablet_loader.cc:96] loaded metadata for tablet 089f8012fcca4b2b97c016dd5e10d3df (table kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339])
20:08:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.605018 22507 tablet_loader.cc:96] loaded metadata for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e (table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9])
20:08:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.605351 22507 tablet_loader.cc:96] loaded metadata for tablet c8d53816408c47109c4d95dae94a247a (table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9])
20:08:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.605604 22507 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:08:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.606465 22507 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.606562 22507 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:08:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.607600 22507 catalog_manager.cc:1504] Loading token signing keys...
20:08:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.608364 22507 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256: Loaded TSK: 0
20:08:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.609210 22507 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:08:30.630 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.189:34501(127.21.11.189:34501)] connecting to peer
20:08:30.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501
20:08:30.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.189:34501(127.21.11.189:34501)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501
20:08:30.631 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.21.11.189:34501(127.21.11.189:34501) for tablet Kudu Master: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501
20:08:30.650 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.190:36769(127.21.11.190:36769)] connecting to peer
20:08:30.651 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.189:34501(127.21.11.189:34501)] connecting to peer
20:08:30.651 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.11.190:36769(127.21.11.190:36769)] Successfully connected to peer
20:08:30.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501
20:08:30.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.189:34501(127.21.11.189:34501)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501
20:08:30.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.189:34501: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501
20:08:30.653 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:08:30.654 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [1ms] received response from server master-127.21.11.189:34501: Network error: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: Network error: Failed to connect to peer master-127.21.11.189:34501(127.21.11.189:34501): Connection refused: /127.21.11.189:34501, [23ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: OK, deferred=Deferred@2058434330(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@219d700c, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@4fb1d4e0)) after lookup
20:08:30.667 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xab0a1c8d, L:/127.0.0.1:53688 - R:/127.21.11.190:36769] using TOKEN/null
20:08:30.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.708053 22445 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Refusing update from remote peer 67c79f7042884b1ba605f75e40717256: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
20:08:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.709161 22505 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
20:08:30.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.711767 21738 consensus_peers.cc:485] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 -> Peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Couldn't send request to peer 773cf88af1d548c89309e0d6a065502c. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:08:30.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.717198 22518 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:30.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.717597 22518 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:30.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.718461 22484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: New leader 67c79f7042884b1ba605f75e40717256. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:30.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.718721 22484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.720916 22503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.721179 22503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 [sys.catalog]: This master's current role is: LEADER
20:08:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.721370 22484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.721562 22484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.727669 21753 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47012:
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:08:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:30.729451 21753 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:08:30.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.896824 22284 heartbeater.cc:347] Connected to a master server at 127.21.11.190:36769
20:08:30.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.898255 22435 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "b8380684688842afb1f56d7d8f29be21" instance_seqno: 1715026107109190) as {username='slave'} at 127.21.11.131:34137; Asking this server to re-register.
20:08:30.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.898806 22000 heartbeater.cc:347] Connected to a master server at 127.21.11.190:36769
20:08:30.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.898834 22284 heartbeater.cc:464] Registering TS with master...
20:08:30.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.900187 22435 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" instance_seqno: 1715026106175533) as {username='slave'} at 127.21.11.129:34215; Asking this server to re-register.
20:08:30.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.900313 22434 ts_manager.cc:194] Registered new tserver with Master: b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:30.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.900931 22000 heartbeater.cc:464] Registering TS with master...
20:08:30.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:30.901856 22434 ts_manager.cc:194] Registered new tserver with Master: 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115)
20:08:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:31.247665 21738 consensus_peers.cc:485] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 -> Peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Couldn't send request to peer 773cf88af1d548c89309e0d6a065502c. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:08:31.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:31.692121 21738 consensus_peers.cc:485] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 -> Peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Couldn't send request to peer 773cf88af1d548c89309e0d6a065502c. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:08:31.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:31.883741 21999 heartbeater.cc:503] Master 127.21.11.188:45949 was elected leader, sending a full tablet report...
20:08:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:31.893662 22283 heartbeater.cc:503] Master 127.21.11.188:45949 was elected leader, sending a full tablet report...
20:08:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.243125 21738 consensus_peers.cc:485] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 -> Peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Couldn't send request to peer 773cf88af1d548c89309e0d6a065502c. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
20:08:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.396821 22140 heartbeater.cc:503] Master 127.21.11.188:45949 was elected leader, sending a full tablet report...
20:08:32.490 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.11.189:34501
20:08:32.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

20:08:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.491690 21550 external_mini_cluster.cc:1326] Running /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6wKKBW/build/asan/bin/kudu
20:08:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data
20:08:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/logs
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/info.pb
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.189:34501
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.189
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33805
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:08:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:08:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.725932 21738 consensus_peers.cc:485] T 00000000000000000000000000000000 P 67c79f7042884b1ba605f75e40717256 -> Peer 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501): Couldn't send request to peer 773cf88af1d548c89309e0d6a065502c. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.189:34501: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
20:08:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.810652 22547 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.810963 22547 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.811081 22547 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818485 22547 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818599 22547 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818650 22547 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818713 22547 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818768 22547 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818850 22547 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.818912 22547 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45863/jwks.json
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.828012 22547 master_runner.cc:386] Master server non-default flags:
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.11.148:36531
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.11.190:36769,127.21.11.189:34501,127.21.11.188:45949
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.11.189:34501
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45863/jwks.json
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/info.pb
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.11.189
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33805
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/logs
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 06 May 2024 19:43:22 UTC on efd1337a2aa3
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 167
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:08:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.829649 22547 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:32.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.831676 22547 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.842445 22553 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.842638 22555 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:32.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:32.843097 22552 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:32.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.843688 22547 server_base.cc:1034] running on GCE node
20:08:32.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.844549 22547 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.845939 22547 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.847129 22547 hybrid_clock.cc:648] HybridClock initialized: now 1715026112847098 us; error 22 us; skew 500 ppm
20:08:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.847458 22547 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:32.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.849638 22547 webserver.cc:460] Webserver started at http://127.21.11.189:33805/ using document root <none> and password file <none>
20:08:32.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.850390 22547 fs_manager.cc:359] Metadata directory not provided
20:08:32.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.850522 22547 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:32.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.855595 22547 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.002s	sys 0.002s
20:08:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.858422 22561 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:08:32.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.859717 22547 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.859891 22547 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data,/tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773cf88af1d548c89309e0d6a065502c"
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-06 20:08:24 on dist-test-slave-zmnq"
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.860646 22547 fs_report.cc:383] FS layout report
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/wal
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/data
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.883805 22547 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:08:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.884650 22547 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.884958 22547 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:32.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.912581 22547 rpc_server.cc:307] RPC server started. Bound to: 127.21.11.189:34501
20:08:32.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.912621 22624 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.11.189:34501 every 8 connection(s)
20:08:32.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.914239 22547 server_base.cc:1166] Dumped server information to /tmp/dist-test-task6wKKBW/test-tmp/mini-kudu-cluster1829609404776008343/master-1/data/info.pb
20:08:32.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.918244 21550 external_mini_cluster.cc:1388] Started /tmp/dist-test-task6wKKBW/build/asan/bin/kudu as pid 22547
20:08:32.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:08:32.918 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.11.188:45949
20:08:32.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

20:08:32.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.919243 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 21720
20:08:32.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.920625 21794 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.921403 22625 sys_catalog.cc:263] Verifying existing consensus state
20:08:32.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.925467 22625 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Bootstrap starting.
20:08:32.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.941445 22625 log.cc:824] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Log is configured to *not* fsync() on all Append() calls
20:08:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.955581 22625 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:08:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.956058 22625 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Bootstrap complete.
20:08:32.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.964088 22625 raft_consensus.cc:348] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:32.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.964946 22625 raft_consensus.cc:729] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 773cf88af1d548c89309e0d6a065502c, State: Initialized, Role: FOLLOWER
20:08:32.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.965608 22625 consensus_queue.cc:260] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:32.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.966545 22629 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:32.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.966905 22629 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: FOLLOWER
20:08:32.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.967687 22625 sys_catalog.cc:564] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: configured and running, proceeding with master startup.
20:08:32.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.990739 22640 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:32.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.990850 22640 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: loading cluster ID for follower catalog manager: success
20:08:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.995147 22640 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: acquiring CA information for follower catalog manager: success
20:08:32.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:32.997620 22640 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:08:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.066649 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 21720
20:08:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.069242 22589 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 3 FOLLOWER]: Advancing to term 4
20:08:33.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.079229 22651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:33.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.079824 22651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: FOLLOWER
20:08:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.080137 22629 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 67c79f7042884b1ba605f75e40717256. Latest consensus state: current_term: 4 leader_uuid: "67c79f7042884b1ba605f75e40717256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.080438 22629 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: FOLLOWER
20:08:33.091 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.11.188:45949(127.21.11.188:45949)] handling channelInactive
20:08:33.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.188:45949(127.21.11.188:45949)] cleaning up while in state READY due to: connection closed
20:08:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.091131 22565 connection.cc:535] server connection from 127.0.0.1:50020 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:33.091 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:08:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.091838 22642 negotiation.cc:337] Failed RPC negotiation. Trace:
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.014464 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.21.11.188:45949 (local address 127.0.0.1:56956)
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.014634 (+   170us) negotiation.cc:107] Waiting for socket to connect
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.014651 (+    17us) client_negotiation.cc:174] Beginning negotiation
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.014734 (+    83us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.066217 (+ 51483us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.066226 (+     9us) client_negotiation.cc:394] Received NEGOTIATE response from server
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.066260 (+    34us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.066754 (+   494us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.066769 (+    15us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.090572 (+ 23803us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":50}
20:08:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.092067 22547 master_runner.cc:186] Error getting master registration for 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107), 
20:08:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.097204 22547 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:08:33.330 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.188:45949(127.21.11.188:45949)] connecting to peer
20:08:33.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.188:45949(127.21.11.188:45949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.331 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.21.11.188:45949(127.21.11.188:45949) for tablet Kudu Master: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.350 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.188:45949(127.21.11.188:45949)] connecting to peer
20:08:33.350 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.189:34501(127.21.11.189:34501)] connecting to peer
20:08:33.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.188:45949(127.21.11.188:45949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.351 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.188:45949: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.351 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.11.189:34501(127.21.11.189:34501)] Successfully connected to peer
20:08:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.352870 22652 raft_consensus.cc:482] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 67c79f7042884b1ba605f75e40717256)
20:08:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.353104 22652 raft_consensus.cc:504] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:33.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.355123 22652 leader_election.cc:288] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 773cf88af1d548c89309e0d6a065502c (127.21.11.189:34501), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.356866 22419 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.358810 22419 leader_election.cc:334] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:33.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.363631 22589 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "773cf88af1d548c89309e0d6a065502c" is_pre_election: true
20:08:33.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.364701 22419 leader_election.cc:302] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa; no voters: 67c79f7042884b1ba605f75e40717256, 773cf88af1d548c89309e0d6a065502c
20:08:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.365072 22652 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:08:33.366 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x16002a29, L:/127.0.0.1:50028 - R:/127.21.11.189:34501] using TOKEN/null
20:08:33.367 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:33.370 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.188:45949(127.21.11.188:45949)] connecting to peer
20:08:33.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.188:45949(127.21.11.188:45949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.188:45949: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.371 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.376772 22629 raft_consensus.cc:482] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 67c79f7042884b1ba605f75e40717256)
20:08:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.376971 22629 raft_consensus.cc:504] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.378500 22629 leader_election.cc:288] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:33.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.378984 22445 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "773cf88af1d548c89309e0d6a065502c" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" is_pre_election: true
20:08:33.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.379194 22562 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:33.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.379243 22445 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 773cf88af1d548c89309e0d6a065502c in term 4.
20:08:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.379942 22562 leader_election.cc:302] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 773cf88af1d548c89309e0d6a065502c; no voters: 
20:08:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.380314 22629 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 4 FOLLOWER]: Leader pre-election won for term 5
20:08:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.380432 22629 raft_consensus.cc:482] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 4 FOLLOWER]: Starting leader election (detected failure of leader 67c79f7042884b1ba605f75e40717256)
20:08:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.380554 22629 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 4 FOLLOWER]: Advancing to term 5
20:08:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.380846 22562 leader_election.cc:334] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:33.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.382515 22629 raft_consensus.cc:504] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.383116 22629 leader_election.cc:288] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 5 election: Requested vote from peers 279092f506ba4af8b3d5dec9ce4ba0aa (127.21.11.190:36769), 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949)
20:08:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.383507 22445 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "773cf88af1d548c89309e0d6a065502c" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa"
20:08:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.383704 22445 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 4 FOLLOWER]: Advancing to term 5
20:08:33.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.384249 22562 leader_election.cc:334] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111)
20:08:33.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.385517 22445 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 773cf88af1d548c89309e0d6a065502c in term 5.
20:08:33.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.385871 22562 leader_election.cc:302] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 279092f506ba4af8b3d5dec9ce4ba0aa, 773cf88af1d548c89309e0d6a065502c; no voters: 67c79f7042884b1ba605f75e40717256
20:08:33.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.386122 22629 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 5 FOLLOWER]: Leader election won for term 5
20:08:33.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.386245 22629 raft_consensus.cc:686] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [term 5 LEADER]: Becoming Leader. State: Replica: 773cf88af1d548c89309e0d6a065502c, State: Running, Role: LEADER
20:08:33.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.386613 22629 consensus_queue.cc:237] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 4.13, Last appended by leader: 13, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } }
20:08:33.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.387778 22651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 773cf88af1d548c89309e0d6a065502c. Latest consensus state: current_term: 5 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:33.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.387933 22651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: LEADER
20:08:33.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.388655 22658 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:08:33.390 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.188:45949(127.21.11.188:45949)] connecting to peer
20:08:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.390882 22658 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339]
20:08:33.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.188:45949(127.21.11.188:45949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.188:45949: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.391479 22658 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9]
20:08:33.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.393088 22658 tablet_loader.cc:96] loaded metadata for tablet 089f8012fcca4b2b97c016dd5e10d3df (table kudu_system.kudu_transactions [id=d316c944530044d588f5f8d1e5d8d339])
20:08:33.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.393507 22658 tablet_loader.cc:96] loaded metadata for tablet b6cb0f0af93e4bcdbfac2ec6c3cfea9e (table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9])
20:08:33.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.393854 22658 tablet_loader.cc:96] loaded metadata for tablet c8d53816408c47109c4d95dae94a247a (table txn_manager_fallback_rolling [id=d8ea12cfb2c84b01925f915dd71058b9])
20:08:33.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.394141 22658 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:08:33.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501; will retry
20:08:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.395154 22658 catalog_manager.cc:1259] Loaded cluster ID: 1b28b360a624438e9d354eeeb3a1aa7e
20:08:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.395233 22658 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:08:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.396183 22658 catalog_manager.cc:1504] Loading token signing keys...
20:08:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.396903 22658 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c: Loaded TSK: 0
20:08:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.397609 22658 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:08:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.404593 22142 heartbeater.cc:347] Connected to a master server at 127.21.11.189:34501
20:08:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.404735 22142 heartbeater.cc:503] Master 127.21.11.189:34501 was elected leader, sending a full tablet report...
20:08:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.407085 22578 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "6c8f731f16954b8c864f9236254cdb02" instance_seqno: 1715026106647160) as {username='slave'} at 127.21.11.130:34409; Asking this server to re-register.
20:08:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.407682 22142 heartbeater.cc:464] Registering TS with master...
20:08:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.407850 22142 heartbeater.cc:511] Master 127.21.11.189:34501 requested a full tablet report, sending...
20:08:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.409088 22578 ts_manager.cc:194] Registered new tserver with Master: 6c8f731f16954b8c864f9236254cdb02 (127.21.11.130:45751)
20:08:33.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.21.11.188:45949(127.21.11.188:45949)] connecting to peer
20:08:33.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.11.188:45949(127.21.11.188:45949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.11.188:45949: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949
20:08:33.412 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:08:33.412 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=82), Traces: [0ms] sending RPC to server master-127.21.11.188:45949, [1ms] received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [23ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [37ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [80ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, deferred=Deferred@1803998792(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@74a50e08 -> (continuation of Deferred@1001611727 after retry RPC after error@357073202) -> (continuation of Deferred@1218523461 after retry RPC after error@346596119) -> (continuation of Deferred@884282708 after retry RPC after error@1696941025), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@244f6d19 -> (continuation of Deferred@1001611727 after retry RPC after error@357073202) -> (continuation of Deferred@1218523461 after retry RPC after error@346596119) -> (continuation of Deferred@884282708 after retry RPC after error@1696941025))) after lookup
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.452804 22578 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50048:
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:08:33.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.456471 22625 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
20:08:33.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.476219 22562 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Couldn't send request to peer 67c79f7042884b1ba605f75e40717256. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:08:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.501806 22445 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [term 5 FOLLOWER]: Refusing update from remote peer 773cf88af1d548c89309e0d6a065502c: Log matching property violated. Preceding OpId in replica: term: 4 index: 13. Preceding OpId from leader: term: 5 index: 14. (index mismatch)
20:08:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.502609 22651 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [LEADER]: Connected to new peer: Peer: permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
20:08:33.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.504587 22652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: New leader 773cf88af1d548c89309e0d6a065502c. Latest consensus state: current_term: 5 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:33.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.504933 22652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:33.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.505952 22629 sys_catalog.cc:455] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:33.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.506186 22629 sys_catalog.cc:458] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c [sys.catalog]: This master's current role is: LEADER
20:08:33.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.506954 22652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "773cf88af1d548c89309e0d6a065502c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "279092f506ba4af8b3d5dec9ce4ba0aa" member_type: VOTER last_known_addr { host: "127.21.11.190" port: 36769 } } peers { permanent_uuid: "773cf88af1d548c89309e0d6a065502c" member_type: VOTER last_known_addr { host: "127.21.11.189" port: 34501 } } peers { permanent_uuid: "67c79f7042884b1ba605f75e40717256" member_type: VOTER last_known_addr { host: "127.21.11.188" port: 45949 } } }
20:08:33.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.507144 22652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 279092f506ba4af8b3d5dec9ce4ba0aa [sys.catalog]: This master's current role is: FOLLOWER
20:08:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.581795 22610 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:50028 (request call id 4) took 168 ms (client timeout 168 ms). Trace:
20:08:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.581949 22610 rpcz_store.cc:269] 0506 20:08:33.413450 (+     0us) service_pool.cc:168] Inserting onto call queue
20:08:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.413549 (+    99us) service_pool.cc:225] Handling call
20:08:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.581726 (+168177us) inbound_call.cc:173] Queueing success response
20:08:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":545,"init.queue_time_us":60,"thread_start_us":990,"threads_started":4}
20:08:33.583 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.21.11.188:45949, [1ms] received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [23ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [37ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [80ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [83ms] sending RPC to server master-127.21.11.189:34501, [83ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1803998792(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@74a50e08 -> (continuation of Deferred@1001611727 after retry RPC after error@357073202) -> (continuation of Deferred@1218523461 after retry RPC after error@346596119) -> (continuation of Deferred@884282708 after retry RPC after error@1696941025) -> (continuation of Deferred@1966002844 after retry RPC@1772975832), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@244f6d19 -> (continuation of Deferred@1001611727 after retry RPC after error@357073202) -> (continuation of Deferred@1218523461 after retry RPC after error@346596119) -> (continuation of Deferred@884282708 after retry RPC after error@1696941025) -> (continuation of Deferred@1966002844 after retry RPC@1772975832)))
20:08:33.583 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.21.11.188:45949, [1ms] received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [23ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [37ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.21.11.188:45949,127.21.11.190:36769,127.21.11.189:34501) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.188:45949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.190:36769, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.11.189:34501, [80ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.188:45949: Network error: Failed to connect to peer master-127.21.11.188:45949(127.21.11.188:45949): Connection refused: /127.21.11.188:45949, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.189:34501: OK, [83ms] sending RPC to server master-127.21.11.189:34501, [83ms] Sub RPC ConnectToMaster: received response from server master-127.21.11.190:36769: OK, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1803998792(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@74a50e08 -> (continuation of Deferred@1001611727 after retry RPC after error@357073202) -> (continuation of Deferred@1218523461 after retry RPC after error@346596119) -> (continuation of Deferred@884282708 after retry RPC after error@1696941025) -> (continuation of Deferred@1966002844 after retry RPC@1772975832), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@244f6d19 -> (continuation of Deferred@1001611727 after retry RPC after error@357073202) -> (continuation of Deferred@1218523461 after retry RPC after error@346596119) -> (continuation of Deferred@884282708 after retry RPC after error@1696941025) -> (continuation of Deferred@1966002844 after retry RPC@1772975832)))
20:08:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.830822 22168 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.013s from last keepalive heartbeat (effective timeout is 1.000s)
20:08:33.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.838258 22201 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.21.11.190:36769)
20:08:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.841379 22610 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:50028 (request call id 5) took 250 ms (client timeout 250 ms). Trace:
20:08:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.841501 22610 rpcz_store.cc:269] 0506 20:08:33.591184 (+     0us) service_pool.cc:168] Inserting onto call queue
20:08:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.591272 (+    88us) service_pool.cc:225] Handling call
20:08:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.841308 (+250036us) inbound_call.cc:173] Queueing success response
20:08:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:08:33.842 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@643308103(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@79a09c30, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@7a190baa))
20:08:33.842 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@643308103(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@79a09c30, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@7a190baa))
20:08:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:33.854033 22673 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
20:08:33.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.904310 22285 heartbeater.cc:347] Connected to a master server at 127.21.11.189:34501
20:08:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.905555 22577 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "b8380684688842afb1f56d7d8f29be21" instance_seqno: 1715026107109190) as {username='slave'} at 127.21.11.131:33653; Asking this server to re-register.
20:08:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.906081 22285 heartbeater.cc:464] Registering TS with master...
20:08:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.906348 22285 heartbeater.cc:511] Master 127.21.11.189:34501 requested a full tablet report, sending...
20:08:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.907610 22577 ts_manager.cc:194] Registered new tserver with Master: b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163)
20:08:33.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.909390 22001 heartbeater.cc:347] Connected to a master server at 127.21.11.189:34501
20:08:33.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.910182 22577 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "0669e5dad664465aacbbeeb3ed37a769" instance_seqno: 1715026106175533) as {username='slave'} at 127.21.11.129:40927; Asking this server to re-register.
20:08:33.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.910635 22001 heartbeater.cc:464] Registering TS with master...
20:08:33.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.910858 22001 heartbeater.cc:511] Master 127.21.11.189:34501 requested a full tablet report, sending...
20:08:33.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:33.911845 22577 ts_manager.cc:194] Registered new tserver with Master: 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115)
20:08:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.013171 22562 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Couldn't send request to peer 67c79f7042884b1ba605f75e40717256. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:08:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.100469 22610 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:50028 (request call id 6) took 250 ms (client timeout 250 ms). Trace:
20:08:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.100589 22610 rpcz_store.cc:269] 0506 20:08:33.850255 (+     0us) service_pool.cc:168] Inserting onto call queue
20:08:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:33.850347 (+    92us) service_pool.cc:225] Handling call
20:08:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:34.100394 (+250047us) inbound_call.cc:173] Queueing success response
20:08:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:08:34.101 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2021702023(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@1eec1ef2, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@63505003))
20:08:34.101 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2021702023(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@1eec1ef2, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@63505003))
20:08:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.360241 22610 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:50028 (request call id 7) took 250 ms (client timeout 250 ms). Trace:
20:08:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.360365 22610 rpcz_store.cc:269] 0506 20:08:34.110120 (+     0us) service_pool.cc:168] Inserting onto call queue
20:08:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:34.110184 (+    64us) service_pool.cc:225] Handling call
20:08:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0506 20:08:34.360176 (+249992us) inbound_call.cc:173] Queueing success response
20:08:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:08:34.361 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1361462168(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@2c1784ba, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@52f21444))
20:08:34.361 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.11.189:34501, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1361462168(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1964073886@2c1784ba, errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/2088991711@52f21444))
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:34.474753 22577 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50060:
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "txn_id"
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "entry_type"
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "identifier"
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "metadata"
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: AUTO_ENCODING
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: DEFAULT_COMPRESSION
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   indirect_data: "<redacted>"""
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "txn_id"
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:08:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.475476 22577 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:08:34.488 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: b8380684688842afb1f56d7d8f29be21 (127.21.11.131:36163): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1715026107 last_transition_timestamp: 1715026113
20:08:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:34.539134 22562 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Couldn't send request to peer 67c79f7042884b1ba605f75e40717256. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:08:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:35.036268 22562 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Couldn't send request to peer 67c79f7042884b1ba605f75e40717256. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
20:08:35.095 [DEBUG - main] (Connection.java:575) [peer master-127.21.11.189:34501(127.21.11.189:34501)] disconnecting while in state READY
20:08:35.095 [DEBUG - main] (Connection.java:575) [peer master-127.21.11.190:36769(127.21.11.190:36769)] disconnecting while in state READY
20:08:35.096 [DEBUG - main] (Connection.java:575) [peer 6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)] disconnecting while in state READY
20:08:35.096 [DEBUG - main] (Connection.java:575) [peer master-127.21.11.188:45949(127.21.11.188:45949)] disconnecting while in state TERMINATED
20:08:35.097 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:08:35.097 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.11.189:34501(127.21.11.189:34501)] handling channelInactive
20:08:35.097 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.189:34501(127.21.11.189:34501)] cleaning up while in state READY due to: connection closed
20:08:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.097873 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 21846
20:08:35.098 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.11.190:36769(127.21.11.190:36769)] handling channelInactive
20:08:35.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.11.190:36769(127.21.11.190:36769)] cleaning up while in state READY due to: connection closed
20:08:35.098 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)] handling channelInactive
20:08:35.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6c8f731f16954b8c864f9236254cdb02(127.21.11.130:45751)] cleaning up while in state READY due to: connection closed
20:08:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.106508 21994 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.199043 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 21846
20:08:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:35.221807 22162 connection.cc:535] client connection to 127.21.11.129:37115 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:35.222052 22162 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.222553 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 22004
20:08:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:35.224432 22162 consensus_peers.cc:485] T 089f8012fcca4b2b97c016dd5e10d3df P b8380684688842afb1f56d7d8f29be21 -> Peer 0669e5dad664465aacbbeeb3ed37a769 (127.21.11.129:37115): Couldn't send request to peer 0669e5dad664465aacbbeeb3ed37a769. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.129:37115: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:08:35.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.229748 22135 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.324507 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 22004
20:08:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.341873 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 22145
20:08:35.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.347924 22278 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.461024 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 22145
20:08:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.479420 22404 master_runner.cc:186] Error getting master registration for 127.21.11.188:45949: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111), 
20:08:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.479655 22404 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:08:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.482491 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 22404
20:08:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.488024 22475 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240506 20:08:35.539454 22562 consensus_peers.cc:485] T 00000000000000000000000000000000 P 773cf88af1d548c89309e0d6a065502c -> Peer 67c79f7042884b1ba605f75e40717256 (127.21.11.188:45949): Couldn't send request to peer 67c79f7042884b1ba605f75e40717256. Status: Network error: Client connection negotiation failed: client connection to 127.21.11.188:45949: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
20:08:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.587337 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 22404
20:08:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.606818 21550 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task6wKKBW/build/asan/bin/kudu pid 22547
20:08:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.612103 22620 generic_service.cc:196] Checking for leaks (request via RPC)
20:08:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240506 20:08:35.719591 21550 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task6wKKBW/build/asan/bin/kudu with pid 22547
20:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-06T20:08:35Z chronyd exiting
20:08:35.864 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
	at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)