21:53:10.751 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:53:10.751 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskSOfzi8/test-tmp
21:53:10.751 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:53:10.751 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskSOfzi8/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:53:10.753 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:53:11.107 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:53:11.107 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:53:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.110946 30867 mini_oidc.cc:152] Starting JWKS server
21:53:11.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.115778 30867 webserver.cc:460] Webserver started at http://127.0.0.1:46781/ using document root <none> and password file <none>
21:53:11.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.116295 30867 mini_oidc.cc:189] Starting OIDC Discovery server
21:53:11.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.118234 30867 webserver.cc:460] Webserver started at http://127.0.0.1:43489/ using document root <none> and password file <none>
21:53:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:53:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
21:53:11.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:53:11Z Disabled control of system clock
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.191661 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/logs
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/info.pb
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.254:43231
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.254
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:53:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:53:11.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.539690 30879 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:11.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.540314 30879 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:11.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.540817 30879 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.574995 30879 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:53:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.575320 30879 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:53:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.575482 30879 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.575687 30879 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:53:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.575899 30879 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:53:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.576097 30879 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:53:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.576244 30879 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46781/jwks.json
21:53:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.613476 30879 master_runner.cc:386] Master server non-default flags:
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.254:43231
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/info.pb
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.254
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/logs
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.614786 30879 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:11.616451 30879 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.632745 30885 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:11.632817 30886 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:12.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:12.774434 30888 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:12.777860 30887 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1146 milliseconds
21:53:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.777992 30879 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:53:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.779400 30879 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.782341 30879 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.783767 30879 hybrid_clock.cc:648] HybridClock initialized: now 1714686792783715 us; error 56 us; skew 500 ppm
21:53:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.784706 30879 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:12.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.792076 30879 webserver.cc:460] Webserver started at http://127.30.36.254:44599/ using document root <none> and password file <none>
21:53:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.793206 30879 fs_manager.cc:359] Metadata directory not provided
21:53:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.793447 30879 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:12.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.793957 30879 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:53:12.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.800254 30879 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/instance:
21:53:12.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf7d7b39d8e464ca8e9abd6da98148a"
21:53:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:12 on dist-test-slave-5c0c"
21:53:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.801775 30879 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal/instance:
21:53:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf7d7b39d8e464ca8e9abd6da98148a"
21:53:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:12 on dist-test-slave-5c0c"
21:53:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.811091 30879 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
21:53:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.817822 30896 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.819279 30879 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.819700 30879 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf7d7b39d8e464ca8e9abd6da98148a"
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:12 on dist-test-slave-5c0c"
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.820096 30879 fs_report.cc:383] FS layout report
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/data
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:12.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.876329 30879 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.878015 30879 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.878584 30879 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:12.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.966751 30879 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.254:43231
21:53:12.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.966818 30957 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.254:43231 every 8 connection(s)
21:53:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.969676 30879 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/info.pb
21:53:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.974992 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 30879
21:53:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.975579 30867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal/instance
21:53:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.976240 30958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.981091 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/logs
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data/info.pb
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.253:39297
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.253
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:53:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:53:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:12.992410 30958 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } has no permanent_uuid. Determining permanent_uuid...
21:53:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.020764 30958 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } has no permanent_uuid. Determining permanent_uuid...
21:53:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.023396 30899 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:13.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.027954 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:13.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.077615 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 1
21:53:13.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.083557 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.152079 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 2
21:53:13.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.156916 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.248481 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 3
21:53:13.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.253036 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.316401 30960 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.317037 30960 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.317543 30960 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.349884 30960 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:53:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.350216 30960 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:53:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.350376 30960 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.350577 30960 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:53:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.350776 30960 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:53:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.350967 30960 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:53:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.351114 30960 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46781/jwks.json
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.387686 30960 master_runner.cc:386] Master server non-default flags:
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.253:39297
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data/info.pb
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.253
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/logs
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:53:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.388978 30960 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.390622 30960 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.396530 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 4
21:53:13.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.402329 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.407544 30969 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.701884 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 5
21:53:13.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.708288 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.255793 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 6
21:53:14.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:14.263224 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:14.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.408388 30972 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:14.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:13.410115 30960 server_base.cc:1034] running on GCE node
21:53:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:13.410475 30970 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.552515 30960 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.556007 30960 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.557536 30960 hybrid_clock.cc:648] HybridClock initialized: now 1714686794557498 us; error 47 us; skew 500 ppm
21:53:14.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.558506 30960 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.565424 30960 webserver.cc:460] Webserver started at http://127.30.36.253:38819/ using document root <none> and password file <none>
21:53:14.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.566713 30960 fs_manager.cc:359] Metadata directory not provided
21:53:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.566931 30960 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.567368 30960 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:53:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.572702 30960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data/instance:
21:53:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d60c154646c46fcbf614e3e3d487be7"
21:53:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:14 on dist-test-slave-5c0c"
21:53:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.573940 30960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal/instance:
21:53:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d60c154646c46fcbf614e3e3d487be7"
21:53:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:14 on dist-test-slave-5c0c"
21:53:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.581677 30960 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
21:53:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.587991 30981 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:14.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.589241 30960 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
21:53:14.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.589582 30960 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal
21:53:14.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d60c154646c46fcbf614e3e3d487be7"
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:14 on dist-test-slave-5c0c"
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.589927 30960 fs_report.cc:383] FS layout report
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data/data
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.646664 30960 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.648223 30960 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.648669 30960 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.733906 30960 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.253:39297
21:53:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.734010 31043 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.253:39297 every 8 connection(s)
21:53:14.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.736608 30960 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/data/info.pb
21:53:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.738981 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 30960
21:53:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.739609 30867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-1/wal/instance
21:53:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.742439 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal
21:53:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/logs
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data/info.pb
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.252:39507
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.252
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:53:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.742663 31045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.754452 31045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } has no permanent_uuid. Determining permanent_uuid...
21:53:14.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.778205 31045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } has no permanent_uuid. Determining permanent_uuid...
21:53:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.792670 31045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } has no permanent_uuid. Determining permanent_uuid...
21:53:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:14.794972 30985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:14.799451 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:14.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.849105 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 1
21:53:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:14.853284 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:14.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:14.921798 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 2
21:53:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:14.926110 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.017700 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 3
21:53:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.022497 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.076303 31047 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:15.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.077102 31047 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.077674 31047 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.113837 31047 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:53:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.114166 31047 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:53:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.114326 31047 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.114531 31047 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:53:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.114728 31047 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:53:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.114924 31047 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:53:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.115072 31047 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46781/jwks.json
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.152760 31047 master_runner.cc:386] Master server non-default flags:
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.252:39507
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data/info.pb
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.252
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/logs
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.154244 31047 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.156083 31047 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.165983 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 4
21:53:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.172061 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.172750 31058 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.323972 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } attempt: 7
21:53:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.337445 30958 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } has no permanent_uuid. Determining permanent_uuid...
21:53:15.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.344080 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.402521 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 1
21:53:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.407274 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.471654 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 5
21:53:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.476322 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.489087 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 2
21:53:15.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.494098 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.580400 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 3
21:53:15.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.587502 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:15.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.728024 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 4
21:53:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.735474 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.019079 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 5
21:53:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.024029 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 6
21:53:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:16.023998 30958 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:16.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:16.035874 31045 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.36.252:39507: Network error: Client connection negotiation failed: client connection to 127.30.36.252:39507: connect: Connection refused (error 111)
21:53:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.173941 31057 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:15.176301 31060 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:15.177114 31047 server_base.cc:1034] running on GCE node
21:53:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.346587 31047 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.350090 31047 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.351593 31047 hybrid_clock.cc:648] HybridClock initialized: now 1714686796351540 us; error 69 us; skew 500 ppm
21:53:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.352470 31047 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.364646 31047 webserver.cc:460] Webserver started at http://127.30.36.252:35549/ using document root <none> and password file <none>
21:53:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.365778 31047 fs_manager.cc:359] Metadata directory not provided
21:53:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.366021 31047 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.366458 31047 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:53:16.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.371799 31047 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data/instance:
21:53:16.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0894e28603564478882b2870f80ff3bb"
21:53:16.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:16 on dist-test-slave-5c0c"
21:53:16.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.373150 31047 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal/instance:
21:53:16.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0894e28603564478882b2870f80ff3bb"
21:53:16.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:16 on dist-test-slave-5c0c"
21:53:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.381377 31047 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
21:53:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.387660 31072 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.388903 31047 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
21:53:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.389299 31047 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal
21:53:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0894e28603564478882b2870f80ff3bb"
21:53:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:16 on dist-test-slave-5c0c"
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.389638 31047 fs_report.cc:383] FS layout report
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data/data
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:16.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.454075 31047 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.455752 31047 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.456214 31047 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:16.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.543262 31047 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.252:39507
21:53:16.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.543363 31134 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.252:39507 every 8 connection(s)
21:53:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.546160 31047 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/data/info.pb
21:53:16.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.550822 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 31047
21:53:16.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.551293 30867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-2/wal/instance
21:53:16.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.551584 31135 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.571300 31135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } has no permanent_uuid. Determining permanent_uuid...
21:53:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.576735 30958 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 6
21:53:16.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.604126 31135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } has no permanent_uuid. Determining permanent_uuid...
21:53:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.614830 30958 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Bootstrap starting.
21:53:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.617123 31135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } has no permanent_uuid. Determining permanent_uuid...
21:53:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.623802 30958 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Neither blocks nor log segments found. Creating new log.
21:53:16.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.627943 30958 log.cc:824] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Log is configured to *not* fsync() on all Append() calls
21:53:16.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.632898 30958 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: No bootstrap required, opened a new log
21:53:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.635960 31135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: Bootstrap starting.
21:53:16.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.643294 31135 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: Neither blocks nor log segments found. Creating new log.
21:53:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.646895 31135 log.cc:824] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: Log is configured to *not* fsync() on all Append() calls
21:53:16.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.652448 31135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: No bootstrap required, opened a new log
21:53:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.660420 30958 raft_consensus.cc:348] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.661293 30958 raft_consensus.cc:374] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.661597 30958 raft_consensus.cc:729] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: caf7d7b39d8e464ca8e9abd6da98148a, State: Initialized, Role: FOLLOWER
21:53:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.662437 30958 consensus_queue.cc:260] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.665380 31147 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.666437 31147 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: FOLLOWER
21:53:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.666774 30958 sys_catalog.cc:564] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: configured and running, proceeding with master startup.
21:53:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.679993 31135 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.680676 31135 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.681114 31135 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0894e28603564478882b2870f80ff3bb, State: Initialized, Role: FOLLOWER
21:53:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.682355 31135 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.688892 31147 raft_consensus.cc:482] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.688259 31156 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.689513 31156 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: FOLLOWER
21:53:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.689457 31147 raft_consensus.cc:504] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.690379 31135 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: configured and running, proceeding with master startup.
21:53:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.692747 31147 leader_election.cc:288] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297), 0894e28603564478882b2870f80ff3bb (127.30.36.252:39507)
21:53:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.694788 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0894e28603564478882b2870f80ff3bb" is_pre_election: true
21:53:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.696166 31099 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate caf7d7b39d8e464ca8e9abd6da98148a in term 0.
21:53:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.698050 30900 leader_election.cc:302] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0894e28603564478882b2870f80ff3bb, caf7d7b39d8e464ca8e9abd6da98148a; no voters:
21:53:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.701118 31162 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:53:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.702761 31147 raft_consensus.cc:2781] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Leader pre-election won for term 1
21:53:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.703146 31147 raft_consensus.cc:482] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:53:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.703495 31147 raft_consensus.cc:3037] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 0 FOLLOWER]: Advancing to term 1
21:53:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:16.711782 31078 tablet.cc:2369] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:53:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.712625 31147 raft_consensus.cc:504] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.718989 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0894e28603564478882b2870f80ff3bb"
21:53:16.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.721968 31099 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 0 FOLLOWER]: Advancing to term 1
21:53:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.728175 31147 leader_election.cc:288] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [CANDIDATE]: Term 1 election: Requested vote from peers 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297), 0894e28603564478882b2870f80ff3bb (127.30.36.252:39507)
21:53:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.733067 31099 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate caf7d7b39d8e464ca8e9abd6da98148a in term 1.
21:53:16.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.735059 30900 leader_election.cc:302] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0894e28603564478882b2870f80ff3bb, caf7d7b39d8e464ca8e9abd6da98148a; no voters:
21:53:16.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.736093 31147 raft_consensus.cc:2781] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 FOLLOWER]: Leader election won for term 1
21:53:16.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.741488 31147 raft_consensus.cc:686] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 LEADER]: Becoming Leader. State: Replica: caf7d7b39d8e464ca8e9abd6da98148a, State: Running, Role: LEADER
21:53:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.742638 31147 consensus_queue.cc:237] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:16.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.753577 31172 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:53:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.757547 31171 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: New leader caf7d7b39d8e464ca8e9abd6da98148a. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:16.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.758512 31171 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: LEADER
21:53:16.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.772677 31174 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
21:53:16.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.801023 31174 catalog_manager.cc:1484] Initializing Kudu cluster ID...
21:53:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.767092 31047 master_runner.cc:186] Error getting master registration for 127.30.36.253:39297: OK, instance_id { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" instance_seqno: 1714686794713876 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:53:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.815310 31047 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.725651 30879 master_runner.cc:186] Error getting master registration for 127.30.36.253:39297: OK, instance_id { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" instance_seqno: 1714686794713876 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:53:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.825183 30879 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:16.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.837285 31099 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Refusing update from remote peer caf7d7b39d8e464ca8e9abd6da98148a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:53:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.845278 31171 consensus_queue.cc:1035] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [LEADER]: Connected to new peer: Peer: permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:16.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.883970 31156 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader caf7d7b39d8e464ca8e9abd6da98148a. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:16.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.884639 31176 mvcc.cc:205] Tried to move back new op lower bound from 7023357119818448896 to 7023357119469711360. Current Snapshot: MvccSnapshot[applied={T|T < 7023357119818448896}]
21:53:16.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.884924 31156 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: FOLLOWER
21:53:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.890110 31147 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.891683 31147 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: LEADER
21:53:16.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.894932 31156 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.895749 31156 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: FOLLOWER
21:53:16.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.897897 31174 catalog_manager.cc:1347] Generated new cluster ID: fa2c7acce0f645c3b6362e195359b353
21:53:16.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.898186 31174 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
21:53:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.932029 31174 catalog_manager.cc:1370] Generated new certificate authority record
21:53:16.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.934360 31174 catalog_manager.cc:1504] Loading token signing keys...
21:53:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.961727 31174 catalog_manager.cc:5857] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Generated new TSK 0
21:53:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.962975 31174 catalog_manager.cc:1514] Initializing in-progress tserver states...
21:53:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:16.987418 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/logs
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data/info.pb
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.193:0
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.36.193
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.193
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:53:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:53:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.096379 31045 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } attempt: 7
21:53:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.137312 31045 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: Bootstrap starting.
21:53:17.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.146086 31045 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: Neither blocks nor log segments found. Creating new log.
21:53:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.148972 31045 log.cc:824] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: Log is configured to *not* fsync() on all Append() calls
21:53:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.161147 31045 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: No bootstrap required, opened a new log
21:53:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.199838 31045 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:17.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.201177 31045 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:17.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.201809 31045 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d60c154646c46fcbf614e3e3d487be7, State: Initialized, Role: FOLLOWER
21:53:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.204573 31045 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:17.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.211257 31045 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: configured and running, proceeding with master startup.
21:53:17.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.212002 31008 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d60c154646c46fcbf614e3e3d487be7" is_pre_election: true
21:53:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.213117 31007 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d60c154646c46fcbf614e3e3d487be7"
21:53:17.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.214198 31007 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:35114 (request call id 2) took 498 ms (client timeout 437 ms). Trace:
21:53:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.214582 31007 rpcz_store.cc:269] 0502 21:53:16.716191 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:53:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 21:53:16.716365 (+ 174us) service_pool.cc:225] Handling call
21:53:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 21:53:17.214106 (+497741us) inbound_call.cc:173] Queueing success response
21:53:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
21:53:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.216405 31005 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d60c154646c46fcbf614e3e3d487be7" is_pre_election: true
21:53:17.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.218816 31004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d60c154646c46fcbf614e3e3d487be7"
21:53:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.227003 31008 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:35114 (request call id 1) took 533 ms (client timeout 409 ms). Trace:
21:53:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.227452 31008 rpcz_store.cc:269] 0502 21:53:16.693256 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:53:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 21:53:16.693467 (+ 211us) service_pool.cc:225] Handling call
21:53:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 21:53:17.226869 (+533402us) inbound_call.cc:173] Queueing success response
21:53:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
21:53:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.242385 31006 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 0 FOLLOWER]: Advancing to term 1
21:53:17.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.257014 31185 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.266041 31185 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: This master's current role is: FOLLOWER
21:53:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.267518 31006 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Refusing update from remote peer caf7d7b39d8e464ca8e9abd6da98148a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:53:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.268934 31185 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.269690 31185 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.283676 30923 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d60c154646c46fcbf614e3e3d487be7" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" is_pre_election: true
21:53:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.284823 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d60c154646c46fcbf614e3e3d487be7" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0894e28603564478882b2870f80ff3bb" is_pre_election: true
21:53:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.287216 30982 leader_election.cc:302] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7d60c154646c46fcbf614e3e3d487be7; no voters: 0894e28603564478882b2870f80ff3bb, caf7d7b39d8e464ca8e9abd6da98148a
21:53:17.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.294607 31187 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:53:17.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.296154 31185 leader_election.cc:288] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231), 0894e28603564478882b2870f80ff3bb (127.30.36.252:39507)
21:53:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.301002 31147 consensus_queue.cc:1035] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:17.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.343431 31194 mvcc.cc:205] Tried to move back new op lower bound from 7023357119818448896 to 7023357119469711360. Current Snapshot: MvccSnapshot[applied={T|T < 7023357119818448896}]
21:53:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.357883 31187 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.358649 31187 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: This master's current role is: FOLLOWER
21:53:17.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.361213 31185 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader caf7d7b39d8e464ca8e9abd6da98148a. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:17.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.362090 31185 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: This master's current role is: FOLLOWER
21:53:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.386547 31201 catalog_manager.cc:1259] Loaded cluster ID: fa2c7acce0f645c3b6362e195359b353
21:53:17.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.387110 31201 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: loading cluster ID for follower catalog manager: success
21:53:17.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.396487 31201 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: acquiring CA information for follower catalog manager: success
21:53:17.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.429632 31201 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:53:17.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.432278 31171 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:17.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.433395 31171 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: LEADER
21:53:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.543401 31181 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.544435 31181 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:17.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.545766 31181 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.618525 31181 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:17.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.619808 31181 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:53:17.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.620333 31181 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:53:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.620942 31181 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:53:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.621812 31181 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.36.193
21:53:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.697950 31181 tablet_server_runner.cc:78] Tablet server non-default flags:
21:53:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.193:0
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data/info.pb
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.193
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.36.193
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/logs
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:17.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.700066 31181 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:17.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.702975 31181 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.743605 31210 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.779842 31172 catalog_manager.cc:1259] Loaded cluster ID: fa2c7acce0f645c3b6362e195359b353
21:53:17.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.780407 31172 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: loading cluster ID for follower catalog manager: success
21:53:17.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.801993 31172 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: acquiring CA information for follower catalog manager: success
21:53:17.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:17.840143 31172 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:53:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:19.124053 31209 debug-util.cc:398] Leaking SignalData structure 0x7b0800014e80 after lost signal to thread 31181
21:53:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:20.861299 31209 kernel_stack_watchdog.cc:198] Thread 31181 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 380ms:
21:53:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:53:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:53:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:53:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:53:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:20.861927 31181 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.120s user 0.710s sys 0.913s
21:53:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:20.862296 31181 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.121s user 0.710s sys 0.913s
21:53:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:17.749800 31211 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:20.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:20.863119 31212 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3114 milliseconds
21:53:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:20.865605 31213 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.866192 31181 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:53:20.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.868973 31181 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.872260 31181 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:20.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.873756 31181 hybrid_clock.cc:648] HybridClock initialized: now 1714686800873632 us; error 136 us; skew 500 ppm
21:53:20.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.875162 31181 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:20.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.884328 31181 webserver.cc:460] Webserver started at http://127.30.36.193:46801/ using document root <none> and password file <none>
21:53:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.885702 31181 fs_manager.cc:359] Metadata directory not provided
21:53:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.886049 31181 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.886662 31181 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:53:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.894443 31181 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data/instance:
21:53:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "063a30eb49f74ffb916a2499b0e0e3b7"
21:53:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:20 on dist-test-slave-5c0c"
21:53:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.896159 31181 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal/instance:
21:53:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "063a30eb49f74ffb916a2499b0e0e3b7"
21:53:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:20 on dist-test-slave-5c0c"
21:53:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.906708 31181 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.002s
21:53:20.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.915114 31223 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.916728 31181 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
21:53:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.917232 31181 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal
21:53:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "063a30eb49f74ffb916a2499b0e0e3b7"
21:53:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:20 on dist-test-slave-5c0c"
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:20.917699 31181 fs_report.cc:383] FS layout report
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data/data
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.026950 31181 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.029114 31181 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.029769 31181 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.048520 31181 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:53:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.048852 31181 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:53:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.049324 31181 ts_tablet_manager.cc:590] Registered 0 tablets
21:53:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.049610 31181 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:53:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.496718 31181 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.193:34085
21:53:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.496894 31354 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.193:34085 every 8 connection(s)
21:53:21.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.500309 31181 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/data/info.pb
21:53:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.504318 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 31181
21:53:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.505017 30867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-0/wal/instance
21:53:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.524196 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal
21:53:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data
21:53:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/logs
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data/info.pb
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.194:0
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.36.194
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.194
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:53:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:53:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:53:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.541103 31355 heartbeater.cc:347] Connected to a master server at 127.30.36.252:39507
21:53:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.541800 31355 heartbeater.cc:464] Registering TS with master...
21:53:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.543524 31355 heartbeater.cc:511] Master 127.30.36.252:39507 requested a full tablet report, sending...
21:53:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.549005 31089 ts_manager.cc:194] Registered new tserver with Master: 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085)
21:53:21.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.569033 31356 heartbeater.cc:347] Connected to a master server at 127.30.36.253:39297
21:53:21.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.569540 31356 heartbeater.cc:464] Registering TS with master...
21:53:21.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.570598 31356 heartbeater.cc:511] Master 127.30.36.253:39297 requested a full tablet report, sending...
21:53:21.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.575738 30998 ts_manager.cc:194] Registered new tserver with Master: 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085)
21:53:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.623095 31357 heartbeater.cc:347] Connected to a master server at 127.30.36.254:43231
21:53:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.623956 31357 heartbeater.cc:464] Registering TS with master...
21:53:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.625571 31357 heartbeater.cc:511] Master 127.30.36.254:43231 requested a full tablet report, sending...
21:53:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.632424 30913 ts_manager.cc:194] Registered new tserver with Master: 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085)
21:53:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:21.636839 30913 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.36.193:38443
21:53:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.035794 31359 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.036403 31359 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.037186 31359 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:22.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.076141 31359 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.077096 31359 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:53:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.077384 31359 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:53:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.077625 31359 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:53:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.078150 31359 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.36.194
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:22.117552 31359 tablet_server_runner.cc:78] Tablet server non-default flags:
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.194:0
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data/info.pb
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.194
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.36.194
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/logs
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:22.119195 31359 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:22.121107 31359 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.144160 31366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:22.642546 31357 heartbeater.cc:503] Master 127.30.36.254:43231 was elected leader, sending a full tablet report...
21:53:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.144227 31367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:22.146304 31369 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:23.428799 31368 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1282 milliseconds
21:53:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.429042 31359 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:53:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.430778 31359 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.434217 31359 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:23.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.435727 31359 hybrid_clock.cc:648] HybridClock initialized: now 1714686803435688 us; error 45 us; skew 500 ppm
21:53:23.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.436583 31359 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.443841 31359 webserver.cc:460] Webserver started at http://127.30.36.194:46121/ using document root <none> and password file <none>
21:53:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.444804 31359 fs_manager.cc:359] Metadata directory not provided
21:53:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.445078 31359 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.445564 31359 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:53:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.450770 31359 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data/instance:
21:53:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "671e1a1320e34abc9062288bf7b1ff79"
21:53:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:23 on dist-test-slave-5c0c"
21:53:23.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.452751 31359 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal/instance:
21:53:23.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "671e1a1320e34abc9062288bf7b1ff79"
21:53:23.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:23 on dist-test-slave-5c0c"
21:53:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.461050 31359 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
21:53:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.467551 31378 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.468966 31359 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
21:53:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.469305 31359 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal
21:53:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "671e1a1320e34abc9062288bf7b1ff79"
21:53:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:23 on dist-test-slave-5c0c"
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.469659 31359 fs_report.cc:383] FS layout report
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data/data
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.540508 31359 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.542534 31359 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.543141 31359 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.561409 31359 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:53:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.561765 31359 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:53:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.562103 31359 ts_tablet_manager.cc:590] Registered 0 tablets
21:53:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.562333 31359 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:53:23.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.803053 31359 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.194:37095
21:53:23.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.803633 31504 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.194:37095 every 8 connection(s)
21:53:23.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.805975 31359 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/data/info.pb
21:53:23.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.806912 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 31359
21:53:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.807471 30867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-1/wal/instance
21:53:23.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.825886 31505 heartbeater.cc:347] Connected to a master server at 127.30.36.252:39507
21:53:23.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.826794 31505 heartbeater.cc:464] Registering TS with master...
21:53:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.829097 31505 heartbeater.cc:511] Master 127.30.36.252:39507 requested a full tablet report, sending...
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.831758 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/logs
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data/info.pb
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.195:0
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.36.195
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.195
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:53:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:53:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.833854 31507 heartbeater.cc:347] Connected to a master server at 127.30.36.254:43231
21:53:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.834280 31507 heartbeater.cc:464] Registering TS with master...
21:53:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.835338 31507 heartbeater.cc:511] Master 127.30.36.254:43231 requested a full tablet report, sending...
21:53:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.835361 31089 ts_manager.cc:194] Registered new tserver with Master: 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.838068 30913 ts_manager.cc:194] Registered new tserver with Master: 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.839677 31506 heartbeater.cc:347] Connected to a master server at 127.30.36.253:39297
21:53:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.840020 31506 heartbeater.cc:464] Registering TS with master...
21:53:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.840062 30913 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.36.194:40293
21:53:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.840914 31506 heartbeater.cc:511] Master 127.30.36.253:39297 requested a full tablet report, sending...
21:53:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:23.844390 30998 ts_manager.cc:194] Registered new tserver with Master: 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:24.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.216051 31510 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:24.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.216594 31510 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:24.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.217285 31510 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.251423 31510 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:24.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.252139 31510 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:53:24.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.252421 31510 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:53:24.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.252657 31510 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:53:24.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.253203 31510 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.36.195
21:53:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:24.292331 31510 tablet_server_runner.cc:78] Tablet server non-default flags:
21:53:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.195:0
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data/info.pb
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.195
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.36.195
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/logs
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:24.294008 31510 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:24.296123 31510 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:24.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.316327 31516 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:24.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:24.844969 31507 heartbeater.cc:503] Master 127.30.36.254:43231 was elected leader, sending a full tablet report...
21:53:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.316931 31517 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:24.317476 31519 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:24.317763 31510 server_base.cc:1034] running on GCE node
21:53:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.494930 31510 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.498621 31510 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.500276 31510 hybrid_clock.cc:648] HybridClock initialized: now 1714686805500222 us; error 40 us; skew 500 ppm
21:53:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.501763 31510 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.510717 31510 webserver.cc:460] Webserver started at http://127.30.36.195:38015/ using document root <none> and password file <none>
21:53:25.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.512718 31510 fs_manager.cc:359] Metadata directory not provided
21:53:25.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.513062 31510 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:25.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.513641 31510 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:53:25.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.519640 31510 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data/instance:
21:53:25.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26bf3d68521d4da68f33cbc12f2bcc38"
21:53:25.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:25 on dist-test-slave-5c0c"
21:53:25.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.520923 31510 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal/instance:
21:53:25.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26bf3d68521d4da68f33cbc12f2bcc38"
21:53:25.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:25 on dist-test-slave-5c0c"
21:53:25.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.529397 31510 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.005s
21:53:25.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.535542 31528 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:25.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.536983 31510 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.007s sys 0.000s
21:53:25.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.537365 31510 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal
21:53:25.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26bf3d68521d4da68f33cbc12f2bcc38"
21:53:25.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:25 on dist-test-slave-5c0c"
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.537755 31510 fs_report.cc:383] FS layout report
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data/data
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.605346 31510 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.607007 31510 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.607533 31510 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.619072 31510 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:53:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.619441 31510 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:53:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.619740 31510 ts_tablet_manager.cc:590] Registered 0 tablets
21:53:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.619900 31510 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:53:25.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.847810 31510 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.195:40399
21:53:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.848377 31654 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.195:40399 every 8 connection(s)
21:53:25.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.850955 31510 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/data/info.pb
21:53:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.861872 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 31510
21:53:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.862392 30867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/ts-2/wal/instance
21:53:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.862323 31657 heartbeater.cc:347] Connected to a master server at 127.30.36.254:43231
21:53:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.865177 31657 heartbeater.cc:464] Registering TS with master...
21:53:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.866853 31657 heartbeater.cc:511] Master 127.30.36.254:43231 requested a full tablet report, sending...
21:53:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.870163 30913 ts_manager.cc:194] Registered new tserver with Master: 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399)
21:53:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.872063 30913 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.36.195:40657
21:53:25.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.874537 31655 heartbeater.cc:347] Connected to a master server at 127.30.36.252:39507
21:53:25.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.875174 31655 heartbeater.cc:464] Registering TS with master...
21:53:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.876017 31655 heartbeater.cc:511] Master 127.30.36.252:39507 requested a full tablet report, sending...
21:53:25.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.879210 31656 heartbeater.cc:347] Connected to a master server at 127.30.36.253:39297
21:53:25.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.879312 31089 ts_manager.cc:194] Registered new tserver with Master: 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399)
21:53:25.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.879551 31656 heartbeater.cc:464] Registering TS with master...
21:53:25.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.880276 31656 heartbeater.cc:511] Master 127.30.36.253:39297 requested a full tablet report, sending...
21:53:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.883088 30998 ts_manager.cc:194] Registered new tserver with Master: 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399)
21:53:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.902328 30867 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
21:53:25.915 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:53:25.915 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:53:25.917 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.36.254"
port: 43231
}
bound_http_address {
host: "127.30.36.254"
port: 44599
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.36.253"
port: 39297
}
bound_http_address {
host: "127.30.36.253"
port: 38819
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.36.252"
port: 39507
}
bound_http_address {
host: "127.30.36.252"
port: 35549
}
}
}
21:53:25.917 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:53:25.918 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.30.36.193"
port: 34085
}
bound_http_address {
host: "127.30.36.193"
port: 46801
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.30.36.194"
port: 37095
}
bound_http_address {
host: "127.30.36.194"
port: 46121
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.30.36.195"
port: 40399
}
bound_http_address {
host: "127.30.36.195"
port: 38015
}
}
}
21:53:25.918 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:53:25.920 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:53:25.922 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:25.924 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.36.252:39507(127.30.36.252:39507)] connecting to peer
21:53:25.924 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.36.254:43231(127.30.36.254:43231)] connecting to peer
21:53:25.925 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.36.253:39297(127.30.36.253:39297)] Successfully connected to peer
21:53:25.925 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.36.252:39507(127.30.36.252:39507)] Successfully connected to peer
21:53:25.926 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.36.254:43231(127.30.36.254:43231)] Successfully connected to peer
21:53:25.929 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.36.253
21:53:25.932 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.36.252
21:53:25.946 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.36.254
21:53:25.951 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2b9137f9, L:/127.0.0.1:46156 - R:/127.30.36.252:39507] using SASL/PLAIN
21:53:25.954 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xca3b2ed3, L:/127.0.0.1:44352 - R:/127.30.36.253:39297] using SASL/PLAIN
21:53:25.971 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x46322459, L:/127.0.0.1:50110 - R:/127.30.36.254:43231] using SASL/PLAIN
21:53:25.974 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:53:25.974 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=53), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [32ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: OK, [51ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, deferred=null) after lookup
21:53:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:25.977808 30913 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50110:
21:53:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
21:53:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:53:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:53:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:25.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:25.983345 30913 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:53:26.034 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 74dfa6585b0f498b955ce7b14f5ce5a8
21:53:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.100262 31440 tablet_service.cc:1467] Processing CreateTablet for tablet 57ebb4042e1e46e6b19291540f93c26a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:53:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.100242 31287 tablet_service.cc:1467] Processing CreateTablet for tablet 57ebb4042e1e46e6b19291540f93c26a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:53:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.100281 31286 tablet_service.cc:1467] Processing CreateTablet for tablet 67c18a9ed4054b569eda34120457d870 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:53:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.102532 31287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 57ebb4042e1e46e6b19291540f93c26a. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.102530 31440 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 57ebb4042e1e46e6b19291540f93c26a. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.100304 31439 tablet_service.cc:1467] Processing CreateTablet for tablet 67c18a9ed4054b569eda34120457d870 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:53:26.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.105783 31286 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 67c18a9ed4054b569eda34120457d870. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.106405 31439 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 67c18a9ed4054b569eda34120457d870. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.115254 31589 tablet_service.cc:1467] Processing CreateTablet for tablet 67c18a9ed4054b569eda34120457d870 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:53:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.117959 31590 tablet_service.cc:1467] Processing CreateTablet for tablet 57ebb4042e1e46e6b19291540f93c26a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:53:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.120530 31590 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 57ebb4042e1e46e6b19291540f93c26a. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.123366 31589 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 67c18a9ed4054b569eda34120457d870. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.147110 31681 tablet_bootstrap.cc:492] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: Bootstrap starting.
21:53:26.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.153453 31682 tablet_bootstrap.cc:492] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7: Bootstrap starting.
21:53:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.159907 31681 tablet_bootstrap.cc:654] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: Neither blocks nor log segments found. Creating new log.
21:53:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.163491 31681 log.cc:824] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: Log is configured to *not* fsync() on all Append() calls
21:53:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.166163 31683 tablet_bootstrap.cc:492] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: Bootstrap starting.
21:53:26.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.167148 31682 tablet_bootstrap.cc:654] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7: Neither blocks nor log segments found. Creating new log.
21:53:26.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.169739 31682 log.cc:824] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7: Log is configured to *not* fsync() on all Append() calls
21:53:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.176812 31682 tablet_bootstrap.cc:492] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7: No bootstrap required, opened a new log
21:53:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.177461 31682 ts_tablet_manager.cc:1367] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7: Time spent bootstrapping tablet: real 0.025s user 0.016s sys 0.003s
21:53:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.179932 31683 tablet_bootstrap.cc:654] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: Neither blocks nor log segments found. Creating new log.
21:53:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.181075 31681 tablet_bootstrap.cc:492] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: No bootstrap required, opened a new log
21:53:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.181655 31681 ts_tablet_manager.cc:1367] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: Time spent bootstrapping tablet: real 0.036s user 0.025s sys 0.004s
21:53:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.182543 31683 log.cc:824] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: Log is configured to *not* fsync() on all Append() calls
21:53:26.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.189178 31683 tablet_bootstrap.cc:492] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: No bootstrap required, opened a new log
21:53:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.189792 31683 ts_tablet_manager.cc:1367] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: Time spent bootstrapping tablet: real 0.024s user 0.010s sys 0.008s
21:53:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.202113 31682 raft_consensus.cc:348] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.202936 31682 raft_consensus.cc:374] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.203250 31682 raft_consensus.cc:729] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 063a30eb49f74ffb916a2499b0e0e3b7, State: Initialized, Role: FOLLOWER
21:53:26.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.204097 31682 consensus_queue.cc:260] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.208669 31682 ts_tablet_manager.cc:1398] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7: Time spent starting tablet: real 0.031s user 0.018s sys 0.012s
21:53:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.211526 31682 tablet_bootstrap.cc:492] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7: Bootstrap starting.
21:53:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.215660 31681 raft_consensus.cc:348] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.216670 31681 raft_consensus.cc:374] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.217005 31681 raft_consensus.cc:729] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 671e1a1320e34abc9062288bf7b1ff79, State: Initialized, Role: FOLLOWER
21:53:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.218014 31681 consensus_queue.cc:260] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.229979 31681 ts_tablet_manager.cc:1398] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: Time spent starting tablet: real 0.048s user 0.043s sys 0.004s
21:53:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.234230 31682 tablet_bootstrap.cc:654] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7: Neither blocks nor log segments found. Creating new log.
21:53:26.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.239331 31681 tablet_bootstrap.cc:492] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79: Bootstrap starting.
21:53:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.246892 31681 tablet_bootstrap.cc:654] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79: Neither blocks nor log segments found. Creating new log.
21:53:26.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.248375 31682 tablet_bootstrap.cc:492] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7: No bootstrap required, opened a new log
21:53:26.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.248904 31682 ts_tablet_manager.cc:1367] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7: Time spent bootstrapping tablet: real 0.038s user 0.011s sys 0.007s
21:53:26.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.249256 31683 raft_consensus.cc:348] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.250873 31683 raft_consensus.cc:374] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.251237 31683 raft_consensus.cc:729] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26bf3d68521d4da68f33cbc12f2bcc38, State: Initialized, Role: FOLLOWER
21:53:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.251890 31682 raft_consensus.cc:348] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.252638 31682 raft_consensus.cc:374] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.252275 31683 consensus_queue.cc:260] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.252954 31682 raft_consensus.cc:729] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 063a30eb49f74ffb916a2499b0e0e3b7, State: Initialized, Role: FOLLOWER
21:53:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.253652 31682 consensus_queue.cc:260] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.254431 31681 tablet_bootstrap.cc:492] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79: No bootstrap required, opened a new log
21:53:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.254881 31681 ts_tablet_manager.cc:1367] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79: Time spent bootstrapping tablet: real 0.016s user 0.007s sys 0.004s
21:53:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.260339 31682 ts_tablet_manager.cc:1398] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7: Time spent starting tablet: real 0.011s user 0.006s sys 0.001s
21:53:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.261371 31681 raft_consensus.cc:348] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.262332 31681 raft_consensus.cc:374] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.262625 31681 raft_consensus.cc:729] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 671e1a1320e34abc9062288bf7b1ff79, State: Initialized, Role: FOLLOWER
21:53:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.263526 31681 consensus_queue.cc:260] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.266698 31681 ts_tablet_manager.cc:1398] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79: Time spent starting tablet: real 0.012s user 0.009s sys 0.000s
21:53:26.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.270257 31683 ts_tablet_manager.cc:1398] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: Time spent starting tablet: real 0.080s user 0.037s sys 0.006s
21:53:26.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.273483 31657 heartbeater.cc:503] Master 127.30.36.254:43231 was elected leader, sending a full tablet report...
21:53:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.276497 31683 tablet_bootstrap.cc:492] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38: Bootstrap starting.
21:53:26.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.278941 31691 raft_consensus.cc:482] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:26.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.280804 31691 raft_consensus.cc:504] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.284657 31691 leader_election.cc:288] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085), 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.299815 31683 tablet_bootstrap.cc:654] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38: Neither blocks nor log segments found. Creating new log.
21:53:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.301846 31460 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "57ebb4042e1e46e6b19291540f93c26a" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "671e1a1320e34abc9062288bf7b1ff79" is_pre_election: true
21:53:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.303335 31460 raft_consensus.cc:2454] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 0.
21:53:26.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.305609 31530 leader_election.cc:302] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26bf3d68521d4da68f33cbc12f2bcc38, 671e1a1320e34abc9062288bf7b1ff79; no voters:
21:53:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.307637 31692 raft_consensus.cc:2781] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:53:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.308331 31692 raft_consensus.cc:482] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:53:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.308775 31692 raft_consensus.cc:3037] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.311511 31309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "57ebb4042e1e46e6b19291540f93c26a" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" is_pre_election: true
21:53:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.313495 31309 raft_consensus.cc:2454] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 0.
21:53:26.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.317739 31683 tablet_bootstrap.cc:492] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38: No bootstrap required, opened a new log
21:53:26.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.318423 31683 ts_tablet_manager.cc:1367] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38: Time spent bootstrapping tablet: real 0.042s user 0.007s sys 0.011s
21:53:26.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.318717 31692 raft_consensus.cc:504] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.320953 31692 leader_election.cc:288] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 election: Requested vote from peers 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085), 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.321851 31309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "57ebb4042e1e46e6b19291540f93c26a" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "063a30eb49f74ffb916a2499b0e0e3b7"
21:53:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.321795 31683 raft_consensus.cc:348] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.322135 31460 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "57ebb4042e1e46e6b19291540f93c26a" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "671e1a1320e34abc9062288bf7b1ff79"
21:53:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.322430 31309 raft_consensus.cc:3037] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.322549 31683 raft_consensus.cc:374] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.322736 31460 raft_consensus.cc:3037] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.322846 31683 raft_consensus.cc:729] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26bf3d68521d4da68f33cbc12f2bcc38, State: Initialized, Role: FOLLOWER
21:53:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.323500 31683 consensus_queue.cc:260] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.329449 31683 ts_tablet_manager.cc:1398] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38: Time spent starting tablet: real 0.011s user 0.004s sys 0.002s
21:53:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.330356 31460 raft_consensus.cc:2454] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 1.
21:53:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.331367 31530 leader_election.cc:302] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26bf3d68521d4da68f33cbc12f2bcc38, 671e1a1320e34abc9062288bf7b1ff79; no voters:
21:53:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:26.332298 31360 tablet.cc:2369] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:53:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.332523 31692 raft_consensus.cc:2781] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 FOLLOWER]: Leader election won for term 1
21:53:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.333300 31692 raft_consensus.cc:686] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 LEADER]: Becoming Leader. State: Replica: 26bf3d68521d4da68f33cbc12f2bcc38, State: Running, Role: LEADER
21:53:26.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.334303 31692 consensus_queue.cc:237] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:26.335497 31509 tablet.cc:2369] T 57ebb4042e1e46e6b19291540f93c26a P 671e1a1320e34abc9062288bf7b1ff79: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:53:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.338375 31309 raft_consensus.cc:2454] T 57ebb4042e1e46e6b19291540f93c26a P 063a30eb49f74ffb916a2499b0e0e3b7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 1.
21:53:26.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.346208 30912 catalog_manager.cc:5484] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 reported cstate change: term changed from 0 to 1, leader changed from <none> to 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195). New cstate: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } health_report { overall_health: HEALTHY } } }
21:53:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:26.363005 31658 tablet.cc:2369] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:53:26.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.377250 31687 raft_consensus.cc:482] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:26.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.377753 31687 raft_consensus.cc:504] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.380347 31687 leader_election.cc:288] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095), 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399)
21:53:26.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.395505 31460 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "67c18a9ed4054b569eda34120457d870" candidate_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "671e1a1320e34abc9062288bf7b1ff79" is_pre_election: true
21:53:26.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.395534 31610 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "67c18a9ed4054b569eda34120457d870" candidate_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" is_pre_election: true
21:53:26.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.396091 31460 raft_consensus.cc:2454] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 063a30eb49f74ffb916a2499b0e0e3b7 in term 0.
21:53:26.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.396543 31610 raft_consensus.cc:2454] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 063a30eb49f74ffb916a2499b0e0e3b7 in term 0.
21:53:26.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.397584 31225 leader_election.cc:302] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 063a30eb49f74ffb916a2499b0e0e3b7, 671e1a1320e34abc9062288bf7b1ff79; no voters:
21:53:26.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.398635 31687 raft_consensus.cc:2781] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:53:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.398924 31687 raft_consensus.cc:482] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:53:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.399245 31687 raft_consensus.cc:3037] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.403710 31687 raft_consensus.cc:504] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.405256 31687 leader_election.cc:288] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [CANDIDATE]: Term 1 election: Requested vote from peers 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095), 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399)
21:53:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.406337 31460 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "67c18a9ed4054b569eda34120457d870" candidate_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "671e1a1320e34abc9062288bf7b1ff79"
21:53:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.406416 31610 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "67c18a9ed4054b569eda34120457d870" candidate_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26bf3d68521d4da68f33cbc12f2bcc38"
21:53:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.407132 31460 raft_consensus.cc:3037] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.407177 31610 raft_consensus.cc:3037] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.414001 31460 raft_consensus.cc:2454] T 67c18a9ed4054b569eda34120457d870 P 671e1a1320e34abc9062288bf7b1ff79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 063a30eb49f74ffb916a2499b0e0e3b7 in term 1.
21:53:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.414026 31610 raft_consensus.cc:2454] T 67c18a9ed4054b569eda34120457d870 P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 063a30eb49f74ffb916a2499b0e0e3b7 in term 1.
21:53:26.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.415335 31227 leader_election.cc:302] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 063a30eb49f74ffb916a2499b0e0e3b7, 26bf3d68521d4da68f33cbc12f2bcc38; no voters:
21:53:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.416088 31687 raft_consensus.cc:2781] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 1 FOLLOWER]: Leader election won for term 1
21:53:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.417773 31687 raft_consensus.cc:686] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [term 1 LEADER]: Becoming Leader. State: Replica: 063a30eb49f74ffb916a2499b0e0e3b7, State: Running, Role: LEADER
21:53:26.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.418601 31687 consensus_queue.cc:237] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.430291 30912 catalog_manager.cc:5484] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193). New cstate: current_term: 1 leader_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } health_report { overall_health: UNKNOWN } } }
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.654713 30912 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50120:
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:53:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:26.657032 30912 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:53:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.691949 31287 tablet_service.cc:1467] Processing CreateTablet for tablet 3e01705f70f04339974abac6cdfd99fa (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:53:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.693763 31287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e01705f70f04339974abac6cdfd99fa. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.693449 31589 tablet_service.cc:1467] Processing CreateTablet for tablet 3e01705f70f04339974abac6cdfd99fa (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:53:26.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.694660 31589 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e01705f70f04339974abac6cdfd99fa. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.698064 31439 tablet_service.cc:1467] Processing CreateTablet for tablet 3e01705f70f04339974abac6cdfd99fa (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:53:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.699848 31439 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e01705f70f04339974abac6cdfd99fa. 1 dirs total, 0 dirs full, 0 dirs failed
21:53:26.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.707768 31682 tablet_bootstrap.cc:492] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: Bootstrap starting.
21:53:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.710388 31683 tablet_bootstrap.cc:492] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: Bootstrap starting.
21:53:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.714172 31682 tablet_bootstrap.cc:654] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: Neither blocks nor log segments found. Creating new log.
21:53:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.716120 31681 tablet_bootstrap.cc:492] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: Bootstrap starting.
21:53:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.716761 31683 tablet_bootstrap.cc:654] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: Neither blocks nor log segments found. Creating new log.
21:53:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.722229 31681 tablet_bootstrap.cc:654] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: Neither blocks nor log segments found. Creating new log.
21:53:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.723091 31682 tablet_bootstrap.cc:492] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: No bootstrap required, opened a new log
21:53:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.723556 31682 ts_tablet_manager.cc:1367] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: Time spent bootstrapping tablet: real 0.016s user 0.010s sys 0.002s
21:53:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.724203 31683 tablet_bootstrap.cc:492] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: No bootstrap required, opened a new log
21:53:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.724632 31683 ts_tablet_manager.cc:1367] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: Time spent bootstrapping tablet: real 0.014s user 0.010s sys 0.001s
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.727344 31683 raft_consensus.cc:348] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.727947 31681 tablet_bootstrap.cc:492] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: No bootstrap required, opened a new log
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.727672 31682 raft_consensus.cc:348] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.728140 31683 raft_consensus.cc:374] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.728427 31681 ts_tablet_manager.cc:1367] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: Time spent bootstrapping tablet: real 0.012s user 0.011s sys 0.000s
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.728413 31682 raft_consensus.cc:374] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.728416 31683 raft_consensus.cc:729] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26bf3d68521d4da68f33cbc12f2bcc38, State: Initialized, Role: FOLLOWER
21:53:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.728682 31682 raft_consensus.cc:729] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 063a30eb49f74ffb916a2499b0e0e3b7, State: Initialized, Role: FOLLOWER
21:53:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.729434 31682 consensus_queue.cc:260] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.729329 31683 consensus_queue.cc:260] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.730840 31692 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.730896 31687 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.731598 31692 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.731664 31687 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.734751 31682 ts_tablet_manager.cc:1398] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: Time spent starting tablet: real 0.010s user 0.003s sys 0.003s
21:53:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.741174 31683 ts_tablet_manager.cc:1398] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: Time spent starting tablet: real 0.016s user 0.003s sys 0.003s
21:53:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.753559 31692 raft_consensus.cc:482] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.754055 31692 raft_consensus.cc:504] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.755972 31692 leader_election.cc:288] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085), 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.756834 31309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e01705f70f04339974abac6cdfd99fa" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" is_pre_election: true
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.757534 31309 raft_consensus.cc:2454] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 0.
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.758951 31460 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e01705f70f04339974abac6cdfd99fa" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "671e1a1320e34abc9062288bf7b1ff79" is_pre_election: true
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.759897 31532 leader_election.cc:302] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 063a30eb49f74ffb916a2499b0e0e3b7, 26bf3d68521d4da68f33cbc12f2bcc38; no voters:
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.760643 31692 raft_consensus.cc:2781] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.761024 31692 raft_consensus.cc:482] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.761340 31692 raft_consensus.cc:3037] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:26.761327 31530 leader_election.cc:341] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095): Illegal state: must be running to vote when last-logged opid is not known
21:53:26.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.764690 31691 consensus_queue.cc:1035] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [LEADER]: Connected to new peer: Peer: permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.766604 31681 raft_consensus.cc:348] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.767468 31681 raft_consensus.cc:374] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:26.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.767762 31681 raft_consensus.cc:729] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 671e1a1320e34abc9062288bf7b1ff79, State: Initialized, Role: FOLLOWER
21:53:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.768549 31681 consensus_queue.cc:260] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.768838 31692 raft_consensus.cc:504] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.770874 31692 leader_election.cc:288] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 election: Requested vote from peers 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085), 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.771955 31460 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e01705f70f04339974abac6cdfd99fa" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "671e1a1320e34abc9062288bf7b1ff79"
21:53:26.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.772555 31460 raft_consensus.cc:3037] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.773797 31308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e01705f70f04339974abac6cdfd99fa" candidate_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "063a30eb49f74ffb916a2499b0e0e3b7"
21:53:26.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.774490 31308 raft_consensus.cc:3037] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 0 FOLLOWER]: Advancing to term 1
21:53:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.779870 31460 raft_consensus.cc:2454] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 1.
21:53:26.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.781188 31530 leader_election.cc:302] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26bf3d68521d4da68f33cbc12f2bcc38, 671e1a1320e34abc9062288bf7b1ff79; no voters:
21:53:26.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.782225 31692 raft_consensus.cc:2781] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 FOLLOWER]: Leader election won for term 1
21:53:26.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.782806 31692 raft_consensus.cc:686] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [term 1 LEADER]: Becoming Leader. State: Replica: 26bf3d68521d4da68f33cbc12f2bcc38, State: Running, Role: LEADER
21:53:26.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.783769 31692 consensus_queue.cc:237] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } }
21:53:26.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.785568 31681 ts_tablet_manager.cc:1398] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: Time spent starting tablet: real 0.057s user 0.013s sys 0.004s
21:53:26.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.787465 31308 raft_consensus.cc:2454] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26bf3d68521d4da68f33cbc12f2bcc38 in term 1.
21:53:26.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.796788 31713 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:26.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.797852 31713 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.818106 30911 catalog_manager.cc:5484] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 reported cstate change: term changed from 0 to 1, leader changed from <none> to 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195). New cstate: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } health_report { overall_health: HEALTHY } } }
21:53:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.862807 31687 consensus_queue.cc:1035] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:53:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.866580 31691 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: TxnStatusTablet state changed. Reason: New leader 26bf3d68521d4da68f33cbc12f2bcc38. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:26.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.867592 31691 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: This TxnStatusTablet replica's current role is: LEADER
21:53:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:26.876797 31532 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.30.36.193:34085, user_credentials={real_user=slave}} blocked reactor thread for 66240.3us
21:53:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.901659 31716 consensus_queue.cc:1035] T 57ebb4042e1e46e6b19291540f93c26a P 26bf3d68521d4da68f33cbc12f2bcc38 [LEADER]: Connected to new peer: Peer: permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.901314 31718 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
21:53:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.915601 31718 txn_status_manager.cc:929] Loading transaction status metadata into memory...
21:53:27.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:26.999683 31730 consensus_queue.cc:1035] T 67c18a9ed4054b569eda34120457d870 P 063a30eb49f74ffb916a2499b0e0e3b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.019654 31459 raft_consensus.cc:1261] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79 [term 1 FOLLOWER]: Refusing update from remote peer 26bf3d68521d4da68f33cbc12f2bcc38: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:53:27.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.021054 31309 raft_consensus.cc:1261] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7 [term 1 FOLLOWER]: Refusing update from remote peer 26bf3d68521d4da68f33cbc12f2bcc38: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:53:27.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.022146 31721 consensus_queue.cc:1035] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [LEADER]: Connected to new peer: Peer: permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:27.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.023065 31717 consensus_queue.cc:1035] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38 [LEADER]: Connected to new peer: Peer: permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:53:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.045910 31730 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: TxnStatusTablet state changed. Reason: New leader 26bf3d68521d4da68f33cbc12f2bcc38. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.046953 31730 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.052903 31713 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: TxnStatusTablet state changed. Reason: New leader 26bf3d68521d4da68f33cbc12f2bcc38. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.053710 31713 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.057041 31722 mvcc.cc:205] Tried to move back new op lower bound from 7023357161533267968 to 7023357160599633920. Current Snapshot: MvccSnapshot[applied={T|T < 7023357161533267968}]
21:53:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.063974 31726 mvcc.cc:205] Tried to move back new op lower bound from 7023357161533267968 to 7023357160599633920. Current Snapshot: MvccSnapshot[applied={T|T < 7023357161533267968}]
21:53:27.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.073854 31692 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:27.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.074685 31692 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: This TxnStatusTablet replica's current role is: LEADER
21:53:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.075866 31692 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.076748 31692 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 26bf3d68521d4da68f33cbc12f2bcc38: This TxnStatusTablet replica's current role is: LEADER
21:53:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.092139 31713 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:27.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.092942 31713 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 671e1a1320e34abc9062288bf7b1ff79: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.097034 31730 tablet_replica.cc:439] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" member_type: VOTER last_known_addr { host: "127.30.36.193" port: 34085 } } peers { permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" member_type: VOTER last_known_addr { host: "127.30.36.194" port: 37095 } } peers { permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" member_type: VOTER last_known_addr { host: "127.30.36.195" port: 40399 } } }
21:53:27.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.097800 31730 tablet_replica.cc:442] T 3e01705f70f04339974abac6cdfd99fa P 063a30eb49f74ffb916a2499b0e0e3b7: This TxnStatusTablet replica's current role is: FOLLOWER
21:53:27.113 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:53:27.119 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 74dfa6585b0f498b955ce7b14f5ce5a8
21:53:27.120 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.36.254:43231
21:53:27.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:53:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.122766 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 30879
21:53:27.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:53:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.169404 31379 connection.cc:535] client connection to 127.30.36.254:43231 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:53:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.170925 31507 heartbeater.cc:644] Failed to heartbeat to 127.30.36.254:43231 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:53:27.171 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.36.254:43231(127.30.36.254:43231)] handling channelInactive
21:53:27.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.254:43231(127.30.36.254:43231)] cleaning up while in state READY due to: connection closed
21:53:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.359042 31739 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Starting pre-election (detected failure of leader caf7d7b39d8e464ca8e9abd6da98148a)
21:53:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.359583 31739 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.361807 31739 leader_election.cc:288] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231), 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297)
21:53:27.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.363850 31006 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0894e28603564478882b2870f80ff3bb" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7d60c154646c46fcbf614e3e3d487be7" is_pre_election: true
21:53:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.364943 31073 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111)
21:53:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.370839 31073 leader_election.cc:334] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111)
21:53:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.371301 31073 leader_election.cc:302] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0894e28603564478882b2870f80ff3bb; no voters: 7d60c154646c46fcbf614e3e3d487be7, caf7d7b39d8e464ca8e9abd6da98148a
21:53:27.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.372069 31739 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:53:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.387248 31742 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader caf7d7b39d8e464ca8e9abd6da98148a)
21:53:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.387974 31742 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.390249 31742 leader_election.cc:288] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231), 0894e28603564478882b2870f80ff3bb (127.30.36.252:39507)
21:53:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.391983 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d60c154646c46fcbf614e3e3d487be7" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0894e28603564478882b2870f80ff3bb" is_pre_election: true
21:53:27.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.393083 30982 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111) [suppressed 6 similar messages]
21:53:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.392849 31099 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7d60c154646c46fcbf614e3e3d487be7 in term 1.
21:53:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.394949 30985 leader_election.cc:302] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0894e28603564478882b2870f80ff3bb, 7d60c154646c46fcbf614e3e3d487be7; no voters:
21:53:27.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.396200 31742 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:53:27.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.396616 31742 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Starting leader election (detected failure of leader caf7d7b39d8e464ca8e9abd6da98148a)
21:53:27.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.397017 31742 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 1 FOLLOWER]: Advancing to term 2
21:53:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.407456 30982 leader_election.cc:334] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111)
21:53:27.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.408448 31742 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:27.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.410494 31742 leader_election.cc:288] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 election: Requested vote from peers caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231), 0894e28603564478882b2870f80ff3bb (127.30.36.252:39507)
21:53:27.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.411600 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d60c154646c46fcbf614e3e3d487be7" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0894e28603564478882b2870f80ff3bb"
21:53:27.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.412197 31099 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 1 FOLLOWER]: Advancing to term 2
21:53:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.414137 30982 leader_election.cc:334] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111)
21:53:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.420820 31099 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7d60c154646c46fcbf614e3e3d487be7 in term 2.
21:53:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.422616 30985 leader_election.cc:302] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0894e28603564478882b2870f80ff3bb, 7d60c154646c46fcbf614e3e3d487be7; no voters: caf7d7b39d8e464ca8e9abd6da98148a
21:53:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.423599 31742 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 2 FOLLOWER]: Leader election won for term 2
21:53:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.427104 31742 raft_consensus.cc:686] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [term 2 LEADER]: Becoming Leader. State: Replica: 7d60c154646c46fcbf614e3e3d487be7, State: Running, Role: LEADER
21:53:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.428596 31742 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.435379 31657 heartbeater.cc:644] Failed to heartbeat to 127.30.36.254:43231 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111)
21:53:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.434906 31745 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7d60c154646c46fcbf614e3e3d487be7. Latest consensus state: current_term: 2 leader_uuid: "7d60c154646c46fcbf614e3e3d487be7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.435815 31745 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: This master's current role is: LEADER
21:53:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.437786 31747 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
21:53:27.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.445986 31747 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]
21:53:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.447458 31747 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816]
21:53:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.452296 31747 tablet_loader.cc:96] loaded metadata for tablet 3e01705f70f04339974abac6cdfd99fa (table kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816])
21:53:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.453505 31747 tablet_loader.cc:96] loaded metadata for tablet 57ebb4042e1e46e6b19291540f93c26a (table txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8])
21:53:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.454491 31747 tablet_loader.cc:96] loaded metadata for tablet 67c18a9ed4054b569eda34120457d870 (table txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8])
21:53:27.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.455294 31747 catalog_manager.cc:1484] Initializing Kudu cluster ID...
21:53:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.458279 31747 catalog_manager.cc:1259] Loaded cluster ID: fa2c7acce0f645c3b6362e195359b353
21:53:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.458468 31747 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
21:53:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.461789 31747 catalog_manager.cc:1504] Loading token signing keys...
21:53:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.464718 31747 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7: Loaded TSK: 0
21:53:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.466753 31747 catalog_manager.cc:1514] Initializing in-progress tserver states...
21:53:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.505393 31357 heartbeater.cc:644] Failed to heartbeat to 127.30.36.254:43231 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111)
21:53:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.531455 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:53:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.545501 31099 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Refusing update from remote peer 7d60c154646c46fcbf614e3e3d487be7: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
21:53:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.547740 31745 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:53:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.553748 31739 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7d60c154646c46fcbf614e3e3d487be7. Latest consensus state: current_term: 2 leader_uuid: "7d60c154646c46fcbf614e3e3d487be7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.554595 31739 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: FOLLOWER
21:53:27.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.568472 31739 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7d60c154646c46fcbf614e3e3d487be7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:27.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.569275 31739 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: FOLLOWER
21:53:27.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.579509 31745 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7d60c154646c46fcbf614e3e3d487be7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.580292 31745 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 [sys.catalog]: This master's current role is: LEADER
21:53:27.624 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.254:43231(127.30.36.254:43231)] connecting to peer
21:53:27.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231
21:53:27.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.254:43231(127.30.36.254:43231)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231
21:53:27.626 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.30.36.254:43231(127.30.36.254:43231) for tablet Kudu Master: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231
21:53:27.643 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.254:43231(127.30.36.254:43231)] connecting to peer
21:53:27.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231
21:53:27.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.254:43231(127.30.36.254:43231)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231
21:53:27.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.254:43231: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231
21:53:27.648 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:53:27.648 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=25), Traces: [0ms] sending RPC to server master-127.30.36.254:43231, [2ms] received response from server master-127.30.36.254:43231: Network error: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231, [18ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [21ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Network error: Failed to connect to peer master-127.30.36.254:43231(127.30.36.254:43231): Connection refused: /127.30.36.254:43231, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: OK, deferred=Deferred@1544324742(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1068816922@65362a9d, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1623565403@4ddb9897)) after lookup
21:53:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:27.699836 30997 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44370:
21:53:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:53:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:53:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:53:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:27.728678 30997 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:53:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:28.043934 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:53:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:28.493111 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:53:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:28.503417 31356 heartbeater.cc:503] Master 127.30.36.253:39297 was elected leader, sending a full tablet report...
21:53:28.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:28.996732 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
21:53:29.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:29.129732 31506 heartbeater.cc:503] Master 127.30.36.253:39297 was elected leader, sending a full tablet report...
21:53:29.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 57ebb4042e1e46e6b19291540f93c26a for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
21:53:29.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 67c18a9ed4054b569eda34120457d870 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
21:53:29.182 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=57ebb4042e1e46e6b19291540f93c26a@[063a30eb49f74ffb916a2499b0e0e3b7(127.30.36.193:34085),26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399)[L],671e1a1320e34abc9062288bf7b1ff79(127.30.36.194:37095)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=67c18a9ed4054b569eda34120457d870@[063a30eb49f74ffb916a2499b0e0e3b7(127.30.36.193:34085)[L],26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399),671e1a1320e34abc9062288bf7b1ff79(127.30.36.194:37095)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:53:29.183 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=5), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.36.253:39297, [4ms] Sub RPC GetTableLocations: received response from server master-127.30.36.253:39297: OK, deferred=null) after lookup
21:53:29.183 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399)] connecting to peer
21:53:29.184 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399)] Successfully connected to peer
21:53:29.203 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x4c50cd85, L:/127.0.0.1:55086 - R:/127.30.36.195:40399] using TOKEN/null
21:53:29.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.213531 31544 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.30.36.254:43231)
21:53:29.310 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.36.254:43231
21:53:29.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:29.312505 30867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/logs
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/info.pb
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.254:43231
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.254
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44599
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:53:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:53:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:29.442086 31656 heartbeater.cc:503] Master 127.30.36.253:39297 was elected leader, sending a full tablet report...
21:53:29.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.497570 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
21:53:29.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.700572 31790 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.701732 31790 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.702487 31790 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.737090 31790 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:53:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.737520 31790 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:53:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.737810 31790 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.738143 31790 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:53:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.738483 31790 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:53:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.738798 31790 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:53:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.739027 31790 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46781/jwks.json
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:29.781818 31790 master_runner.cc:386] Master server non-default flags:
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.36.212:39169
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.36.254:43231,127.30.36.253:39297,127.30.36.252:39507
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.36.254:43231
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46781/jwks.json
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/info.pb
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.36.254
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44599
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/logs
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:29.783449 31790 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:29.785250 31790 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:29.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.808420 31798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:30.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:30.004436 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
21:53:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:30.539553 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
21:53:31.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.032590 30982 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7d60c154646c46fcbf614e3e3d487be7 -> Peer caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231): Couldn't send request to peer caf7d7b39d8e464ca8e9abd6da98148a. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.254:43231: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
21:53:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.808467 31799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:29.809506 31802 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.048816 31800 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:53:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.048902 31790 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:53:31.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.054795 31790 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.057950 31790 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:31.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.059664 31790 hybrid_clock.cc:648] HybridClock initialized: now 1714686811059631 us; error 52 us; skew 500 ppm
21:53:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.060781 31790 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:31.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.068567 31790 webserver.cc:460] Webserver started at http://127.30.36.254:44599/ using document root <none> and password file <none>
21:53:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.069669 31790 fs_manager.cc:359] Metadata directory not provided
21:53:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.069936 31790 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.079329 31790 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
21:53:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.084982 31815 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:53:31.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.086337 31790 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
21:53:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.086746 31790 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data,/tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf7d7b39d8e464ca8e9abd6da98148a"
21:53:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:53:12 on dist-test-slave-5c0c"
21:53:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.089356 31790 fs_report.cc:383] FS layout report
21:53:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/wal
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/data
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:31.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.190896 31790 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:53:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.192553 31790 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.193065 31790 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:31.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.295472 31790 rpc_server.cc:307] RPC server started. Bound to: 127.30.36.254:43231
21:53:31.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.295548 31880 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.36.254:43231 every 8 connection(s)
21:53:31.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.298594 31790 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSOfzi8/test-tmp/mini-kudu-cluster974476936934245107/master-0/data/info.pb
21:53:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.301453 30867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu as pid 31790
21:53:31.302 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:53:31.303 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.36.253:39297
21:53:31.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:53:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.304008 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 30960
21:53:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.314527 31881 sys_catalog.cc:263] Verifying existing consensus state
21:53:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.320804 31881 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Bootstrap starting.
21:53:31.343 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.36.253:39297(127.30.36.253:39297)] handling channelInactive
21:53:31.343 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state READY due to: connection closed
21:53:31.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:53:31.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.368758 31881 log.cc:824] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Log is configured to *not* fsync() on all Append() calls
21:53:31.382 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.383 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.383 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.30.36.253:39297(127.30.36.253:39297) for tablet Kudu Master: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.402 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.403 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.254:43231(127.30.36.254:43231)] connecting to peer
21:53:31.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.404 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.404 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.36.254:43231(127.30.36.254:43231)] Successfully connected to peer
21:53:31.404 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.408375 31881 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:53:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.409353 31881 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: Bootstrap complete.
21:53:31.414 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.36.254
21:53:31.433 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x70435561, L:/127.0.0.1:50910 - R:/127.30.36.254:43231] using SASL/PLAIN
21:53:31.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.254:43231: Catalog manager is not initialized. State: Starting
21:53:31.438 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.439165 31881 raft_consensus.cc:348] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.441918 31881 raft_consensus.cc:729] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: caf7d7b39d8e464ca8e9abd6da98148a, State: Initialized, Role: FOLLOWER
21:53:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.442682 31881 consensus_queue.cc:260] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:31.443 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.445 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.254:43231: Catalog manager is not initialized. State: Starting
21:53:31.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.445922 31885 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.446871 31885 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: FOLLOWER
21:53:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.447646 31881 sys_catalog.cc:564] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: configured and running, proceeding with master startup.
21:53:31.447 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.463 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.254:43231: Catalog manager is not initialized. State: Starting
21:53:31.474 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.479553 31657 heartbeater.cc:347] Connected to a master server at 127.30.36.254:43231
21:53:31.483 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.494920 31835 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "26bf3d68521d4da68f33cbc12f2bcc38" instance_seqno: 1714686805807024) as {username='slave'} at 127.30.36.195:57943; Asking this server to re-register.
21:53:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.498487 31898 catalog_manager.cc:1259] Loaded cluster ID: fa2c7acce0f645c3b6362e195359b353
21:53:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.498718 31898 catalog_manager.cc:1552] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: loading cluster ID for follower catalog manager: success
21:53:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.502053 31657 heartbeater.cc:464] Registering TS with master...
21:53:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.505748 31898 catalog_manager.cc:1574] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: acquiring CA information for follower catalog manager: success
21:53:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.506807 31835 ts_manager.cc:194] Registered new tserver with Master: 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399)
21:53:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.511940 31898 catalog_manager.cc:1602] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:53:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.522785 31901 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 7d60c154646c46fcbf614e3e3d487be7)
21:53:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.523391 31901 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.525633 31901 leader_election.cc:288] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231), 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297)
21:53:31.536 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.562 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.583 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.584 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.584 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.590 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.591 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=208), Traces: [0ms] sending RPC to server master-127.30.36.253:39297, [1ms] received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [24ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [55ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [56ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [63ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [64ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [65ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [88ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [91ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [91ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [92ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [103ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [108ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [153ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [154ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [186ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [187ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [202ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [207ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [208ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, deferred=Deferred@980084078(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1068816922@1fb040f9 -> (continuation of Deferred@738621961 after retry RPC after error@1185275348) -> (continuation of Deferred@730403323 after retry RPC after error@916667387) -> (continuation of Deferred@694868244 after retry RPC after error@1103751824) -> (continuation of Deferred@573324699 after retry RPC after error@226469574) -> (continuation of Deferred@1923251204 after retry RPC after error@2092008874), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1623565403@3f32bf42 -> (continuation of Deferred@738621961 after retry RPC after error@1185275348) -> (continuation of Deferred@730403323 after retry RPC after error@916667387) -> (continuation of Deferred@694868244 after retry RPC after error@1103751824) -> (continuation of Deferred@573324699 after retry RPC after error@226469574) -> (continuation of Deferred@1923251204 after retry RPC after error@2092008874)))
21:53:31.592 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=208), Traces: [0ms] sending RPC to server master-127.30.36.253:39297, [1ms] received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [22ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [24ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [55ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [56ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [63ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [64ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [65ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [88ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [91ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [91ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [92ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [103ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [108ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [153ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [154ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [186ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [187ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [202ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [207ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [208ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, deferred=Deferred@980084078(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1068816922@1fb040f9 -> (continuation of Deferred@738621961 after retry RPC after error@1185275348) -> (continuation of Deferred@730403323 after retry RPC after error@916667387) -> (continuation of Deferred@694868244 after retry RPC after error@1103751824) -> (continuation of Deferred@573324699 after retry RPC after error@226469574) -> (continuation of Deferred@1923251204 after retry RPC after error@2092008874), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1623565403@3f32bf42 -> (continuation of Deferred@738621961 after retry RPC after error@1185275348) -> (continuation of Deferred@730403323 after retry RPC after error@916667387) -> (continuation of Deferred@694868244 after retry RPC after error@1103751824) -> (continuation of Deferred@573324699 after retry RPC after error@226469574) -> (continuation of Deferred@1923251204 after retry RPC after error@2092008874)))
21:53:31.622 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.628 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.630836 31075 leader_election.cc:334] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297): Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:31.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.636917 31357 heartbeater.cc:347] Connected to a master server at 127.30.36.254:43231
21:53:31.643 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.645938 31835 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "063a30eb49f74ffb916a2499b0e0e3b7" instance_seqno: 1714686801441098) as {username='slave'} at 127.30.36.193:33441; Asking this server to re-register.
21:53:31.649 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.648514 31357 heartbeater.cc:464] Registering TS with master...
21:53:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.621809 31845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0894e28603564478882b2870f80ff3bb" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" is_pre_election: true
21:53:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.651079 31845 raft_consensus.cc:2454] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0894e28603564478882b2870f80ff3bb in term 1.
21:53:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.590601 31790 master_runner.cc:186] Error getting master registration for 127.30.36.253:39297: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111),
21:53:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.652611 31790 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.652581 31073 leader_election.cc:302] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0894e28603564478882b2870f80ff3bb, caf7d7b39d8e464ca8e9abd6da98148a; no voters: 7d60c154646c46fcbf614e3e3d487be7
21:53:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.653825 31901 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Leader pre-election won for term 3
21:53:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.653999 31835 ts_manager.cc:194] Registered new tserver with Master: 063a30eb49f74ffb916a2499b0e0e3b7 (127.30.36.193:34085)
21:53:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.654187 31901 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Starting leader election (detected failure of leader 7d60c154646c46fcbf614e3e3d487be7)
21:53:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.654507 31901 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 2 FOLLOWER]: Advancing to term 3
21:53:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.660073 31901 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:31.664 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.663830 31845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0894e28603564478882b2870f80ff3bb" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "caf7d7b39d8e464ca8e9abd6da98148a"
21:53:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.664497 31845 raft_consensus.cc:3037] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 1 FOLLOWER]: Advancing to term 3
21:53:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.669868 31901 leader_election.cc:288] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 3 election: Requested vote from peers caf7d7b39d8e464ca8e9abd6da98148a (127.30.36.254:43231), 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297)
21:53:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.674350 31075 leader_election.cc:334] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297): Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111)
21:53:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.678144 31845 raft_consensus.cc:2454] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0894e28603564478882b2870f80ff3bb in term 3.
21:53:31.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.679982 31073 leader_election.cc:302] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0894e28603564478882b2870f80ff3bb, caf7d7b39d8e464ca8e9abd6da98148a; no voters: 7d60c154646c46fcbf614e3e3d487be7
21:53:31.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.681331 31901 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 3 FOLLOWER]: Leader election won for term 3
21:53:31.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.683565 31901 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [term 3 LEADER]: Becoming Leader. State: Replica: 0894e28603564478882b2870f80ff3bb, State: Running, Role: LEADER
21:53:31.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.685556 31901 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } }
21:53:31.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.692592 31909 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0894e28603564478882b2870f80ff3bb. Latest consensus state: current_term: 3 leader_uuid: "0894e28603564478882b2870f80ff3bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:31.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.693712 31909 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: LEADER
21:53:31.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.696717 31911 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
21:53:31.703 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.705 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.705 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.706187 31911 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8]
21:53:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.709285 31911 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816]
21:53:31.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.714326 31911 tablet_loader.cc:96] loaded metadata for tablet 3e01705f70f04339974abac6cdfd99fa (table kudu_system.kudu_transactions [id=7c3539ce466d49eca93cab319e5d6816])
21:53:31.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.715687 31911 tablet_loader.cc:96] loaded metadata for tablet 57ebb4042e1e46e6b19291540f93c26a (table txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8])
21:53:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.716814 31911 tablet_loader.cc:96] loaded metadata for tablet 67c18a9ed4054b569eda34120457d870 (table txn_manager_fallback_rolling [id=74dfa6585b0f498b955ce7b14f5ce5a8])
21:53:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.717756 31911 catalog_manager.cc:1484] Initializing Kudu cluster ID...
21:53:31.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231; will retry
21:53:31.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.721410 31911 catalog_manager.cc:1259] Loaded cluster ID: fa2c7acce0f645c3b6362e195359b353
21:53:31.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.721664 31911 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
21:53:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.725426 31911 catalog_manager.cc:1504] Loading token signing keys...
21:53:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.728603 31911 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb: Loaded TSK: 0
21:53:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.730141 31911 catalog_manager.cc:1514] Initializing in-progress tserver states...
21:53:31.743 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.30.36.253:39297(127.30.36.253:39297)] connecting to peer
21:53:31.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.253:39297(127.30.36.253:39297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.36.253:39297: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297
21:53:31.750 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:53:31.751 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=128), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [3ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [5ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [5ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [23ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [26ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [41ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [43ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [50ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [67ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [68ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [83ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [86ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [97ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [97ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [123ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [126ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [127ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, deferred=Deferred@1619438949(state=PENDING, result=null, callback=(continuation of Deferred@222397344 after retry RPC after error@1914516843) -> (continuation of Deferred@888884010 after retry RPC after error@1631790421) -> (continuation of Deferred@1612641886 after retry RPC after error@2066948443) -> (continuation of Deferred@360679180 after retry RPC after error@1835950459), errback=(continuation of Deferred@222397344 after retry RPC after error@1914516843) -> (continuation of Deferred@888884010 after retry RPC after error@1631790421) -> (continuation of Deferred@1612641886 after retry RPC after error@2066948443) -> (continuation of Deferred@360679180 after retry RPC after error@1835950459))) after lookup
21:53:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.797770 31845 raft_consensus.cc:1261] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [term 3 FOLLOWER]: Refusing update from remote peer 0894e28603564478882b2870f80ff3bb: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
21:53:31.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.800292 31909 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [LEADER]: Connected to new peer: Peer: permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.001s
21:53:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.804575 31075 consensus_peers.cc:485] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb -> Peer 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297): Couldn't send request to peer 7d60c154646c46fcbf614e3e3d487be7. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.819990 31088 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46180:
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:53:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.824715 31088 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:53:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.855702 31885 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0894e28603564478882b2870f80ff3bb. Latest consensus state: current_term: 3 leader_uuid: "0894e28603564478882b2870f80ff3bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.856503 31885 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: FOLLOWER
21:53:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.871675 31885 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "0894e28603564478882b2870f80ff3bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.873265 31885 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: FOLLOWER
21:53:31.883 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [3ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [5ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [5ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [23ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [26ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [41ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [43ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [50ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [67ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [68ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [83ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [86ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [97ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [97ms] delaying RPC due to: Service unavailable: Master config (127.30.36.253:39297,127.30.36.252:39507,127.30.36.254:43231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.253:39297, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.252:39507, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.36.254:43231, [123ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.253:39297: Network error: Failed to connect to peer master-127.30.36.253:39297(127.30.36.253:39297): Connection refused: /127.30.36.253:39297, [126ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.254:43231: OK, [127ms] Sub RPC ConnectToMaster: received response from server master-127.30.36.252:39507: OK, [129ms] sending RPC to server master-127.30.36.252:39507, deferred=Deferred@1619438949(state=PENDING, result=null, callback=(continuation of Deferred@222397344 after retry RPC after error@1914516843) -> (continuation of Deferred@888884010 after retry RPC after error@1631790421) -> (continuation of Deferred@1612641886 after retry RPC after error@2066948443) -> (continuation of Deferred@360679180 after retry RPC after error@1835950459) -> (continuation of Deferred@1923223602 after retry RPC@536574158), errback=(continuation of Deferred@222397344 after retry RPC after error@1914516843) -> (continuation of Deferred@888884010 after retry RPC after error@1631790421) -> (continuation of Deferred@1612641886 after retry RPC after error@2066948443) -> (continuation of Deferred@360679180 after retry RPC after error@1835950459) -> (continuation of Deferred@1923223602 after retry RPC@536574158)))
21:53:31.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.886240 31901 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "0894e28603564478882b2870f80ff3bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:31.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.887676 31901 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb [sys.catalog]: This master's current role is: LEADER
21:53:31.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.889734 31916 meta_cache.cc:302] tablet 3e01705f70f04339974abac6cdfd99fa: replica 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399) has failed: Timed out: connection negotiation to 127.30.36.195:40399 for RPC CoordinateTransaction timed out after 0.029s (ON_OUTBOUND_QUEUE)
21:53:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.892210 31923 sys_catalog.cc:455] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "0894e28603564478882b2870f80ff3bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "caf7d7b39d8e464ca8e9abd6da98148a" member_type: VOTER last_known_addr { host: "127.30.36.254" port: 43231 } } peers { permanent_uuid: "7d60c154646c46fcbf614e3e3d487be7" member_type: VOTER last_known_addr { host: "127.30.36.253" port: 39297 } } peers { permanent_uuid: "0894e28603564478882b2870f80ff3bb" member_type: VOTER last_known_addr { host: "127.30.36.252" port: 39507 } } }
21:53:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.896238 31119 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:46156 (request call id 13) took 143 ms (client timeout 121 ms). Trace:
21:53:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.893188 31923 sys_catalog.cc:458] T 00000000000000000000000000000000 P caf7d7b39d8e464ca8e9abd6da98148a [sys.catalog]: This master's current role is: FOLLOWER
21:53:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.896673 31119 rpcz_store.cc:269] 0502 21:53:31.752796 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:53:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 21:53:31.753250 (+ 454us) service_pool.cc:225] Handling call
21:53:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 21:53:31.895986 (+142736us) inbound_call.cc:173] Queueing success response
21:53:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":1186,"init.queue_time_us":3114,"init.run_cpu_time_us":22639,"init.run_wall_time_us":80363,"thread_start_us":11292,"threads_started":5}
21:53:31.899 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=0, TimeoutTracker(timeout=0, elapsed=16), Traces: [0ms] received response from server master-127.30.36.252:39507: Timed out: Failed to write to server: (no server available): CoordinateTransaction(txn_status_tablet_id: "3e01705f70f04339974abac6cdfd99fa" op { type: KEEP_TXN_ALIVE txn_id: 0 user: "slave" }) passed its deadline: Timed out: connection negotiation to 127.30.36.195:40399 for RPC CoordinateTransaction timed out after 0.029s (ON_OUTBOUND_QUEUE) (error 0), deferred=null) with no deferred attached!
21:53:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:31.963193 31537 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.021s from last keepalive heartbeat (effective timeout is 1.000s)
21:53:31.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.967396 31548 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.30.36.253:39297)
21:53:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:31.977869 31928 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
21:53:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:32.199469 31507 heartbeater.cc:347] Connected to a master server at 127.30.36.254:43231
21:53:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:32.203579 31835 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "671e1a1320e34abc9062288bf7b1ff79" instance_seqno: 1714686803766444) as {username='slave'} at 127.30.36.194:47877; Asking this server to re-register.
21:53:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:32.205154 31507 heartbeater.cc:464] Registering TS with master...
21:53:32.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:32.209089 31835 ts_manager.cc:194] Registered new tserver with Master: 671e1a1320e34abc9062288bf7b1ff79 (127.30.36.194:37095)
21:53:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:32.308918 31075 consensus_peers.cc:485] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb -> Peer 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297): Couldn't send request to peer 7d60c154646c46fcbf614e3e3d487be7. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:53:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:32.393082 31075 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111) [suppressed 8 similar messages]
21:53:32.431 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 26bf3d68521d4da68f33cbc12f2bcc38 (127.30.36.195:40399): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1714686806 last_transition_timestamp: 1714686811
21:53:32.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:32.782825 31075 consensus_peers.cc:485] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb -> Peer 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297): Couldn't send request to peer 7d60c154646c46fcbf614e3e3d487be7. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:53:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:33.156370 31505 heartbeater.cc:503] Master 127.30.36.252:39507 was elected leader, sending a full tablet report...
21:53:33.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:53:33.311758 31075 consensus_peers.cc:485] T 00000000000000000000000000000000 P 0894e28603564478882b2870f80ff3bb -> Peer 7d60c154646c46fcbf614e3e3d487be7 (127.30.36.253:39297): Couldn't send request to peer 7d60c154646c46fcbf614e3e3d487be7. Status: Network error: Client connection negotiation failed: client connection to 127.30.36.253:39297: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
21:53:33.354 [DEBUG - main] (Connection.java:575) [peer master-127.30.36.253:39297(127.30.36.253:39297)] disconnecting while in state TERMINATED
21:53:33.354 [DEBUG - main] (Connection.java:575) [peer master-127.30.36.252:39507(127.30.36.252:39507)] disconnecting while in state READY
21:53:33.354 [DEBUG - main] (Connection.java:575) [peer master-127.30.36.254:43231(127.30.36.254:43231)] disconnecting while in state READY
21:53:33.355 [DEBUG - main] (Connection.java:575) [peer 26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399)] disconnecting while in state READY
21:53:33.356 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.36.254:43231(127.30.36.254:43231)] handling channelInactive
21:53:33.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.36.254:43231(127.30.36.254:43231)] cleaning up while in state READY due to: connection closed
21:53:33.356 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:53:33.358 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.36.252:39507(127.30.36.252:39507)] handling channelInactive
21:53:33.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.36.252:39507(127.30.36.252:39507)] cleaning up while in state READY due to: connection closed
21:53:33.358 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399)] handling channelInactive
21:53:33.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 26bf3d68521d4da68f33cbc12f2bcc38(127.30.36.195:40399)] cleaning up while in state READY due to: connection closed
21:53:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:33.359462 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 31181
21:53:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:33.400566 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 31359
21:53:33.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:33.443761 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 31510
21:53:33.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:33.493774 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 31790
21:53:33.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:53:33.538106 30867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSOfzi8/build/tsan/bin/kudu with pid 31047
21:53:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:53:34Z chronyd exiting
21:53:35.730 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)