07:02:00.218 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:02:00.218 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskjQXd4I/test-tmp
07:02:00.218 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
07:02:00.219 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskjQXd4I/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
07:02:00.220 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:02:00.589 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:02:00.590 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:02:00.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:00.592607 10714 mini_oidc.cc:152] Starting JWKS server
07:02:00.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:00.595695 10714 webserver.cc:460] Webserver started at http://127.0.0.1:43301/ using document root <none> and password file <none>
07:02:00.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:00.595960 10714 mini_oidc.cc:189] Starting OIDC Discovery server
07:02:00.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:00.596623 10714 webserver.cc:460] Webserver started at http://127.0.0.1:43625/ using document root <none> and password file <none>
07:02:00.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:02:00Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:02:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:02:00Z Disabled control of system clock
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:00.629398 10714 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/logs
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data/info.pb
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.190:40067
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.190
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43301/jwks.json
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:02:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:02:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.982584 10725 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.982915 10725 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.983000 10725 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990003 10725 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990113 10725 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990146 10725 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990190 10725 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990226 10725 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990269 10725 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:02:00.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:00.990300 10725 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43301/jwks.json
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.000065 10725 master_runner.cc:386] Master server non-default flags:
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.190:40067
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43301/jwks.json
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data/info.pb
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.190
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/logs
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.001591 10725 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.003528 10725 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:01.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.014606 10733 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.015153 10730 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.015746 10731 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.016428 10725 server_base.cc:1034] running on GCE node
07:02:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.017192 10725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.018853 10725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.020043 10725 hybrid_clock.cc:648] HybridClock initialized: now 1714114921019992 us; error 53 us; skew 500 ppm
07:02:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.020397 10725 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:01.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.028261 10725 webserver.cc:460] Webserver started at http://127.10.118.190:43003/ using document root <none> and password file <none>
07:02:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.029368 10725 fs_manager.cc:359] Metadata directory not provided
07:02:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.029575 10725 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.030054 10725 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.033699 10725 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data/instance:
07:02:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df97c70ed96443ac833c3a62b3385d37"
07:02:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.034683 10725 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal/instance:
07:02:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df97c70ed96443ac833c3a62b3385d37"
07:02:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.041191 10725 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
07:02:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.046032 10739 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.047972 10725 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.048173 10725 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df97c70ed96443ac833c3a62b3385d37"
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.048408 10725 fs_report.cc:383] FS layout report
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data/data
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.072695 10725 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.073832 10725 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.074180 10725 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.097024 10725 rpc_server.cc:307] RPC server started. Bound to: 127.10.118.190:40067
07:02:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.097052 10800 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.118.190:40067 every 8 connection(s)
07:02:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.098929 10725 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/data/info.pb
07:02:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.105005 10801 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.109046 10714 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 10725
07:02:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.109359 10714 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-0/wal/instance
07:02:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.111706 10714 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/logs
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data/info.pb
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.189:44731
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.189
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43301/jwks.json
07:02:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:02:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:02:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.115207 10801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } has no permanent_uuid. Determining permanent_uuid...
07:02:01.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.136085 10801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } has no permanent_uuid. Determining permanent_uuid...
07:02:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.137022 10741 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.118.189:44731: connect: Connection refused (error 111)
07:02:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.138401 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.189:44731: Network error: Client connection negotiation failed: client connection to 127.10.118.189:44731: connect: Connection refused (error 111)
07:02:01.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.187815 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } attempt: 1
07:02:01.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.189793 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.189:44731: Network error: Client connection negotiation failed: client connection to 127.10.118.189:44731: connect: Connection refused (error 111)
07:02:01.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.258040 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } attempt: 2
07:02:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.260089 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.189:44731: Network error: Client connection negotiation failed: client connection to 127.10.118.189:44731: connect: Connection refused (error 111)
07:02:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.351470 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } attempt: 3
07:02:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.353538 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.189:44731: Network error: Client connection negotiation failed: client connection to 127.10.118.189:44731: connect: Connection refused (error 111)
07:02:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.466038 10803 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.466385 10803 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.466511 10803 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473394 10803 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473507 10803 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473558 10803 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473618 10803 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473671 10803 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473737 10803 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:02:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.473785 10803 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43301/jwks.json
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.482372 10803 master_runner.cc:386] Master server non-default flags:
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal
07:02:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.189:44731
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43301/jwks.json
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data/info.pb
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.189
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/logs
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:01.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.484413 10803 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.486547 10803 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.496779 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } attempt: 4
07:02:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.497936 10811 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.498374 10812 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.499032 10814 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.499056 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.189:44731: Network error: Client connection negotiation failed: client connection to 127.10.118.189:44731: connect: Connection refused (error 111)
07:02:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.499660 10803 server_base.cc:1034] running on GCE node
07:02:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.500541 10803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.502252 10803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.503441 10803 hybrid_clock.cc:648] HybridClock initialized: now 1714114921503378 us; error 53 us; skew 500 ppm
07:02:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.503822 10803 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.506172 10803 webserver.cc:460] Webserver started at http://127.10.118.189:38535/ using document root <none> and password file <none>
07:02:01.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.506992 10803 fs_manager.cc:359] Metadata directory not provided
07:02:01.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.507084 10803 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:01.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.507392 10803 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.510005 10803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data/instance:
07:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "446ba19691fd4c469d75f3459ec45ccd"
07:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.510716 10803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal/instance:
07:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "446ba19691fd4c469d75f3459ec45ccd"
07:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.515978 10803 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
07:02:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.519567 10820 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.521009 10803 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.521147 10803 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "446ba19691fd4c469d75f3459ec45ccd"
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.521317 10803 fs_report.cc:383] FS layout report
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data/data
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:01.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.535622 10803 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.536520 10803 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.536759 10803 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.557802 10881 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.118.189:44731 every 8 connection(s)
07:02:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.557817 10803 rpc_server.cc:307] RPC server started. Bound to: 127.10.118.189:44731
07:02:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.559351 10803 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/data/info.pb
07:02:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.563230 10714 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 10803
07:02:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.563432 10714 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-1/wal/instance
07:02:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.563611 10882 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.564435 10714 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/logs
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data/info.pb
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.188:37597
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.188
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43301/jwks.json
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:02:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:02:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.570057 10882 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } has no permanent_uuid. Determining permanent_uuid...
07:02:01.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.583830 10882 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } has no permanent_uuid. Determining permanent_uuid...
07:02:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.592314 10882 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
07:02:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.593338 10824 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.594805 10882 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.644133 10882 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 1
07:02:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.646193 10882 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.714496 10882 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 2
07:02:01.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.716588 10882 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.798316 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } attempt: 5
07:02:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.804792 10801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
07:02:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.806854 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.807785 10882 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 3
07:02:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.809473 10882 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.858136 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 1
07:02:01.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.860100 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.915004 10884 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.915380 10884 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.915478 10884 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922188 10884 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922312 10884 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922345 10884 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922390 10884 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922425 10884 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922472 10884 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:02:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.922500 10884 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43301/jwks.json
07:02:01.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.928404 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 2
07:02:01.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.930172 10801 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.932133 10884 master_runner.cc:386] Master server non-default flags:
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.188:37597
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43301/jwks.json
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data/info.pb
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.188
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/logs
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.933637 10884 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:01.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.935721 10884 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:01.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.947116 10892 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.947557 10893 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.948159 10895 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.948863 10884 server_base.cc:1034] running on GCE node
07:02:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.949646 10884 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:01.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.951279 10884 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.952463 10884 hybrid_clock.cc:648] HybridClock initialized: now 1714114921952416 us; error 50 us; skew 500 ppm
07:02:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.952667 10882 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 4
07:02:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.952795 10884 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:01.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:01.954532 10882 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.118.188:37597: Network error: Client connection negotiation failed: client connection to 127.10.118.188:37597: connect: Connection refused (error 111)
07:02:01.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.955569 10884 webserver.cc:460] Webserver started at http://127.10.118.188:42221/ using document root <none> and password file <none>
07:02:01.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.956421 10884 fs_manager.cc:359] Metadata directory not provided
07:02:01.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.956511 10884 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:01.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.956784 10884 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.959619 10884 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data/instance:
07:02:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feca55a551e141dd8fff0d68430dcc15"
07:02:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.960430 10884 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal/instance:
07:02:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feca55a551e141dd8fff0d68430dcc15"
07:02:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.966311 10884 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
07:02:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.970214 10901 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:01.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.971829 10884 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.971988 10884 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feca55a551e141dd8fff0d68430dcc15"
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:01 on dist-test-slave-z59g"
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.972167 10884 fs_report.cc:383] FS layout report
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data/data
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.992786 10884 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.993741 10884 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:01.993989 10884 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:02.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.015851 10884 rpc_server.cc:307] RPC server started. Bound to: 127.10.118.188:37597
07:02:02.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.015887 10962 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.118.188:37597 every 8 connection(s)
07:02:02.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.017416 10884 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/data/info.pb
07:02:02.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.022121 10963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:02.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.023663 10714 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 10884
07:02:02.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.023852 10714 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/master-2/wal/instance
07:02:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.031090 10963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } has no permanent_uuid. Determining permanent_uuid...
07:02:02.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.036396 10801 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 3
07:02:02.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.047572 10963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } has no permanent_uuid. Determining permanent_uuid...
07:02:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.055855 10801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: Bootstrap starting.
07:02:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.056969 10963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
07:02:02.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.060317 10801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: Neither blocks nor log segments found. Creating new log.
07:02:02.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.061497 10801 log.cc:824] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: Log is configured to *not* fsync() on all Append() calls
07:02:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.066704 10801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: No bootstrap required, opened a new log
07:02:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.068460 10963 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: Bootstrap starting.
07:02:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.072576 10963 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: Neither blocks nor log segments found. Creating new log.
07:02:02.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.073609 10801 raft_consensus.cc:348] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.073827 10963 log.cc:824] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: Log is configured to *not* fsync() on all Append() calls
07:02:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.073910 10801 raft_consensus.cc:374] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.073997 10801 raft_consensus.cc:729] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: df97c70ed96443ac833c3a62b3385d37, State: Initialized, Role: FOLLOWER
07:02:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.074831 10801 consensus_queue.cc:260] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.076586 10973 sys_catalog.cc:455] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.077072 10973 sys_catalog.cc:458] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.077549 10745 tablet.cc:2369] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:02:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.077597 10963 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: No bootstrap required, opened a new log
07:02:02.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.078049 10801 sys_catalog.cc:564] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: configured and running, proceeding with master startup.
07:02:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.087654 10963 raft_consensus.cc:348] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.088390 10963 raft_consensus.cc:374] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.088675 10963 raft_consensus.cc:729] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: feca55a551e141dd8fff0d68430dcc15, State: Initialized, Role: FOLLOWER
07:02:02.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.089774 10963 consensus_queue.cc:260] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.090706 10984 catalog_manager.cc:1558] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:02:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.090983 10984 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:02:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.092002 10986 sys_catalog.cc:455] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.092761 10986 sys_catalog.cc:458] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.093529 10963 sys_catalog.cc:564] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: configured and running, proceeding with master startup.
07:02:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.090847 10725 master_runner.cc:186] Error getting master registration for 127.10.118.189:44731: OK, instance_id { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" instance_seqno: 1714114921552787 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:02:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.102937 10725 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:02:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.123323 10973 raft_consensus.cc:482] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.123677 10973 raft_consensus.cc:504] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.126086 10973 leader_election.cc:288] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 446ba19691fd4c469d75f3459ec45ccd (127.10.118.189:44731), feca55a551e141dd8fff0d68430dcc15 (127.10.118.188:37597)
07:02:02.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.128454 10928 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "df97c70ed96443ac833c3a62b3385d37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "feca55a551e141dd8fff0d68430dcc15" is_pre_election: true
07:02:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.129361 10928 raft_consensus.cc:2454] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df97c70ed96443ac833c3a62b3385d37 in term 0.
07:02:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.121203 10884 master_runner.cc:186] Error getting master registration for 127.10.118.189:44731: OK, instance_id { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" instance_seqno: 1714114921552787 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:02:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.130084 10884 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:02:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.130266 10743 leader_election.cc:302] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: df97c70ed96443ac833c3a62b3385d37, feca55a551e141dd8fff0d68430dcc15; no voters:
07:02:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.130887 10973 raft_consensus.cc:2781] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.131069 10973 raft_consensus.cc:482] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.131208 10973 raft_consensus.cc:3037] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 0 FOLLOWER]: Advancing to term 1
07:02:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.134065 10997 catalog_manager.cc:1558] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:02:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.134215 10997 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:02:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.134909 10973 raft_consensus.cc:504] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.136727 10973 leader_election.cc:288] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [CANDIDATE]: Term 1 election: Requested vote from peers 446ba19691fd4c469d75f3459ec45ccd (127.10.118.189:44731), feca55a551e141dd8fff0d68430dcc15 (127.10.118.188:37597)
07:02:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.136847 10928 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "df97c70ed96443ac833c3a62b3385d37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "feca55a551e141dd8fff0d68430dcc15"
07:02:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.137084 10928 raft_consensus.cc:3037] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 0 FOLLOWER]: Advancing to term 1
07:02:02.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.140707 10928 raft_consensus.cc:2454] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df97c70ed96443ac833c3a62b3385d37 in term 1.
07:02:02.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.141326 10743 leader_election.cc:302] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: df97c70ed96443ac833c3a62b3385d37, feca55a551e141dd8fff0d68430dcc15; no voters:
07:02:02.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.141897 10973 raft_consensus.cc:2781] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 1 FOLLOWER]: Leader election won for term 1
07:02:02.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.148917 10973 raft_consensus.cc:686] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [term 1 LEADER]: Becoming Leader. State: Replica: df97c70ed96443ac833c3a62b3385d37, State: Running, Role: LEADER
07:02:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.149699 10973 consensus_queue.cc:237] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.162536 10999 sys_catalog.cc:455] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: SysCatalogTable state changed. Reason: New leader df97c70ed96443ac833c3a62b3385d37. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.163905 10999 sys_catalog.cc:458] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: This master's current role is: LEADER
07:02:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.165674 11002 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:02:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.168232 11002 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:02:02.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.185837 10928 raft_consensus.cc:1261] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Refusing update from remote peer df97c70ed96443ac833c3a62b3385d37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.187804 10999 consensus_queue.cc:1035] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.208989 10986 sys_catalog.cc:455] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: SysCatalogTable state changed. Reason: New leader df97c70ed96443ac833c3a62b3385d37. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.209334 10986 sys_catalog.cc:458] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.215811 10999 sys_catalog.cc:455] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.216114 10999 sys_catalog.cc:458] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: This master's current role is: LEADER
07:02:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.219540 11002 catalog_manager.cc:1347] Generated new cluster ID: 1ccf4cff07bf472c83291437d4679755
07:02:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.219892 11002 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:02:02.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.236634 10986 sys_catalog.cc:455] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.236892 10986 sys_catalog.cc:458] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.243134 11002 catalog_manager.cc:1370] Generated new certificate authority record
07:02:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.245882 11002 catalog_manager.cc:1504] Loading token signing keys...
07:02:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.253815 10882 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } attempt: 5
07:02:02.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.268870 10882 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: Bootstrap starting.
07:02:02.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.276484 11002 catalog_manager.cc:5857] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37: Generated new TSK 0
07:02:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.279932 11002 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:02:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.281723 10882 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: Neither blocks nor log segments found. Creating new log.
07:02:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.283368 10882 log.cc:824] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: Log is configured to *not* fsync() on all Append() calls
07:02:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.288096 10882 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: No bootstrap required, opened a new log
07:02:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.299094 10882 raft_consensus.cc:348] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.299825 10882 raft_consensus.cc:374] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:02.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.300307 10882 raft_consensus.cc:729] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 446ba19691fd4c469d75f3459ec45ccd, State: Initialized, Role: FOLLOWER
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.303869 10714 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/logs
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data/info.pb
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.129:0
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.118.129
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.129
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:02:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:02:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:02:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.304922 10882 consensus_queue.cc:260] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.319586 11009 sys_catalog.cc:455] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.320220 11009 sys_catalog.cc:458] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.323066 10846 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "df97c70ed96443ac833c3a62b3385d37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "446ba19691fd4c469d75f3459ec45ccd"
07:02:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.323688 10846 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 0 FOLLOWER]: Advancing to term 1
07:02:02.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.327153 10846 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df97c70ed96443ac833c3a62b3385d37 in term 1.
07:02:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.327548 10847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "df97c70ed96443ac833c3a62b3385d37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "446ba19691fd4c469d75f3459ec45ccd" is_pre_election: true
07:02:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.329232 10845 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Refusing update from remote peer df97c70ed96443ac833c3a62b3385d37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.330780 10882 sys_catalog.cc:564] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: configured and running, proceeding with master startup.
07:02:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.330904 10973 consensus_queue.cc:1035] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.384421 11009 sys_catalog.cc:455] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: SysCatalogTable state changed. Reason: New leader df97c70ed96443ac833c3a62b3385d37. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.384752 11009 sys_catalog.cc:458] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.385370 11016 sys_catalog.cc:455] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.385885 11016 sys_catalog.cc:458] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: This master's current role is: FOLLOWER
07:02:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.416308 11027 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:02:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.423593 10973 sys_catalog.cc:455] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "df97c70ed96443ac833c3a62b3385d37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.423913 10973 sys_catalog.cc:458] T 00000000000000000000000000000000 P df97c70ed96443ac833c3a62b3385d37 [sys.catalog]: This master's current role is: LEADER
07:02:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.808120 11010 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.808468 11010 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.808586 11010 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.815927 11010 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.816084 11010 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:02:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.816121 11010 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:02:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.816171 11010 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:02:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.816236 11010 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.118.129
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.825929 11010 tablet_server_runner.cc:78] Tablet server non-default flags:
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.129:0
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data/info.pb
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.129
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.118.129
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/logs
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.828071 11010 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.830902 11010 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.845710 11033 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.846204 11035 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:02.846837 11032 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:02.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.847517 11010 server_base.cc:1034] running on GCE node
07:02:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.848244 11010 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:02.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.849308 11010 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:02.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.850512 11010 hybrid_clock.cc:648] HybridClock initialized: now 1714114922850433 us; error 70 us; skew 500 ppm
07:02:02.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.850858 11010 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.853426 11010 webserver.cc:460] Webserver started at http://127.10.118.129:39603/ using document root <none> and password file <none>
07:02:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.854269 11010 fs_manager.cc:359] Metadata directory not provided
07:02:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.854357 11010 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.854667 11010 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.857705 11010 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data/instance:
07:02:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d4ed4894d994d448aa50bfaf76ab885"
07:02:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:02 on dist-test-slave-z59g"
07:02:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.858415 11010 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal/instance:
07:02:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d4ed4894d994d448aa50bfaf76ab885"
07:02:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:02 on dist-test-slave-z59g"
07:02:02.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.864197 11010 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.008s sys 0.000s
07:02:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.868192 11041 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.869834 11010 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.869987 11010 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d4ed4894d994d448aa50bfaf76ab885"
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:02 on dist-test-slave-z59g"
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.870191 11010 fs_report.cc:383] FS layout report
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data/data
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.898766 11010 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:02.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.899704 11010 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:02.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.899930 11010 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:02.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.902705 11010 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:02:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.902947 11010 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:02:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.903107 11010 ts_tablet_manager.cc:590] Registered 0 tablets
07:02:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.903291 11010 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:02:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.960045 11010 rpc_server.cc:307] RPC server started. Bound to: 127.10.118.129:37281
07:02:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.960075 11165 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.118.129:37281 every 8 connection(s)
07:02:02.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.961514 11010 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/data/info.pb
07:02:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.965098 11167 heartbeater.cc:347] Connected to a master server at 127.10.118.190:40067
07:02:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.966943 11167 heartbeater.cc:464] Registering TS with master...
07:02:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.967967 11167 heartbeater.cc:511] Master 127.10.118.190:40067 requested a full tablet report, sending...
07:02:02.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.970065 10714 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 11010
07:02:02.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.970292 10714 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-0/wal/instance
07:02:02.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.970499 10755 ts_manager.cc:194] Registered new tserver with Master: 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281)
07:02:02.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.971791 11168 heartbeater.cc:347] Connected to a master server at 127.10.118.189:44731
07:02:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.971946 11168 heartbeater.cc:464] Registering TS with master...
07:02:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.972429 11168 heartbeater.cc:511] Master 127.10.118.189:44731 requested a full tablet report, sending...
07:02:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.972491 10755 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.118.129:44419
07:02:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.974287 11166 heartbeater.cc:347] Connected to a master server at 127.10.118.188:37597
07:02:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.974512 11166 heartbeater.cc:464] Registering TS with master...
07:02:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.974766 10837 ts_manager.cc:194] Registered new tserver with Master: 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281)
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.975574 10714 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/logs
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data/info.pb
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.130:0
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.118.130
07:02:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.130
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.975642 11166 heartbeater.cc:511] Master 127.10.118.188:37597 requested a full tablet report, sending...
07:02:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:02.977339 10918 ts_manager.cc:194] Registered new tserver with Master: 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281)
07:02:03.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.136536 10997 catalog_manager.cc:1259] Loaded cluster ID: 1ccf4cff07bf472c83291437d4679755
07:02:03.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.136646 10997 catalog_manager.cc:1552] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: loading cluster ID for follower catalog manager: success
07:02:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.139165 10997 catalog_manager.cc:1574] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: acquiring CA information for follower catalog manager: success
07:02:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.141006 10997 catalog_manager.cc:1602] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:02:03.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.320626 11171 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:03.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.320930 11171 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:03.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.321089 11171 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.328184 11171 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.328374 11171 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:02:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.328429 11171 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:02:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.328495 11171 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:02:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.328598 11171 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.118.130
07:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.338507 11171 tablet_server_runner.cc:78] Tablet server non-default flags:
07:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.130:0
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data/info.pb
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.130
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.118.130
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/logs
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.340312 11171 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.342284 11171 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.355276 11176 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.355875 11177 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.356192 11179 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.356842 11171 server_base.cc:1034] running on GCE node
07:02:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.357911 11171 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.359354 11171 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.360594 11171 hybrid_clock.cc:648] HybridClock initialized: now 1714114923360510 us; error 80 us; skew 500 ppm
07:02:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.361018 11171 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.364475 11171 webserver.cc:460] Webserver started at http://127.10.118.130:35099/ using document root <none> and password file <none>
07:02:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.365476 11171 fs_manager.cc:359] Metadata directory not provided
07:02:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.365574 11171 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.365900 11171 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.368963 11171 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data/instance:
07:02:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:03 on dist-test-slave-z59g"
07:02:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.369663 11171 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal/instance:
07:02:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:03 on dist-test-slave-z59g"
07:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.375461 11171 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
07:02:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.379606 11185 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.381275 11171 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.381458 11171 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:03 on dist-test-slave-z59g"
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.381659 11171 fs_report.cc:383] FS layout report
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal
07:02:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data/data
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:03.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.397166 11171 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:03.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.398113 11171 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:03.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.398397 11171 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.401422 11171 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:02:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.401522 11171 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:02:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.401641 11171 ts_tablet_manager.cc:590] Registered 0 tablets
07:02:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.401707 11171 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:02:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.423303 11027 catalog_manager.cc:1259] Loaded cluster ID: 1ccf4cff07bf472c83291437d4679755
07:02:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.423689 11027 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: loading cluster ID for follower catalog manager: success
07:02:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.428501 11027 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: acquiring CA information for follower catalog manager: success
07:02:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.432298 11027 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:02:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.487740 11171 rpc_server.cc:307] RPC server started. Bound to: 127.10.118.130:39113
07:02:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.487798 11307 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.118.130:39113 every 8 connection(s)
07:02:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.489145 11171 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/data/info.pb
07:02:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.491441 11309 heartbeater.cc:347] Connected to a master server at 127.10.118.190:40067
07:02:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.491688 11309 heartbeater.cc:464] Registering TS with master...
07:02:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.492815 11309 heartbeater.cc:511] Master 127.10.118.190:40067 requested a full tablet report, sending...
07:02:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.494024 10714 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 11171
07:02:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.494215 10714 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-1/wal/instance
07:02:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.494504 10756 ts_manager.cc:194] Registered new tserver with Master: 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.495752 10756 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.118.130:37269
07:02:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.504822 11308 heartbeater.cc:347] Connected to a master server at 127.10.118.188:37597
07:02:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.505004 11308 heartbeater.cc:464] Registering TS with master...
07:02:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.506433 11308 heartbeater.cc:511] Master 127.10.118.188:37597 requested a full tablet report, sending...
07:02:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.507858 10918 ts_manager.cc:194] Registered new tserver with Master: 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.509699 10714 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/logs
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data/info.pb
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.131:0
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.118.131
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.131
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:03.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:02:03.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:02:03.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:02:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.521703 11310 heartbeater.cc:347] Connected to a master server at 127.10.118.189:44731
07:02:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.521857 11310 heartbeater.cc:464] Registering TS with master...
07:02:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.522126 11310 heartbeater.cc:511] Master 127.10.118.189:44731 requested a full tablet report, sending...
07:02:03.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.523051 10837 ts_manager.cc:194] Registered new tserver with Master: 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.885836 11313 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.886234 11313 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.886421 11313 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:03.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.893679 11313 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.893893 11313 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:02:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.893951 11313 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:02:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.894033 11313 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:02:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.894163 11313 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.118.131
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.903023 11313 tablet_server_runner.cc:78] Tablet server non-default flags:
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.118.148:42101
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.118.131:0
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data/info.pb
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.118.131
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.118.190:40067,127.10.118.189:44731,127.10.118.188:37597
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.118.131
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/logs
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.904881 11313 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:03.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.907073 11313 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.920115 11318 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.920634 11321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:03.921267 11319 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.921938 11313 server_base.cc:1034] running on GCE node
07:02:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.922657 11313 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:03.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.923861 11313 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.925062 11313 hybrid_clock.cc:648] HybridClock initialized: now 1714114923925005 us; error 70 us; skew 500 ppm
07:02:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.925666 11313 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:03.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.928241 11313 webserver.cc:460] Webserver started at http://127.10.118.131:41399/ using document root <none> and password file <none>
07:02:03.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.929188 11313 fs_manager.cc:359] Metadata directory not provided
07:02:03.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.929323 11313 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:03.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.929622 11313 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.933435 11313 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data/instance:
07:02:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b6156414e84176a7e601627eb4e882"
07:02:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:03 on dist-test-slave-z59g"
07:02:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.934371 11313 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal/instance:
07:02:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b6156414e84176a7e601627eb4e882"
07:02:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:03 on dist-test-slave-z59g"
07:02:03.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.940587 11313 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
07:02:03.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.944727 11327 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:03.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.946497 11313 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
07:02:03.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.946694 11313 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b6156414e84176a7e601627eb4e882"
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:03 on dist-test-slave-z59g"
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.946938 11313 fs_report.cc:383] FS layout report
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data/data
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:03.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.965529 11313 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.966543 11313 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.966837 11313 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.969616 11313 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:02:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.969712 11313 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:02:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.969837 11313 ts_tablet_manager.cc:590] Registered 0 tablets
07:02:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.969905 11313 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:02:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:03.978109 11167 heartbeater.cc:503] Master 127.10.118.190:40067 was elected leader, sending a full tablet report...
07:02:04.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.027943 11313 rpc_server.cc:307] RPC server started. Bound to: 127.10.118.131:45339
07:02:04.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.027971 11448 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.118.131:45339 every 8 connection(s)
07:02:04.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.029268 11313 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/data/info.pb
07:02:04.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.032260 11450 heartbeater.cc:347] Connected to a master server at 127.10.118.190:40067
07:02:04.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.032966 11450 heartbeater.cc:464] Registering TS with master...
07:02:04.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.034096 11450 heartbeater.cc:511] Master 127.10.118.190:40067 requested a full tablet report, sending...
07:02:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.035756 10756 ts_manager.cc:194] Registered new tserver with Master: 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.035953 10714 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 11313
07:02:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.036149 10714 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7738263088932212582/ts-2/wal/instance
07:02:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.036805 10756 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.118.131:51501
07:02:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.037751 11449 heartbeater.cc:347] Connected to a master server at 127.10.118.188:37597
07:02:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.037890 11449 heartbeater.cc:464] Registering TS with master...
07:02:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.038374 11449 heartbeater.cc:511] Master 127.10.118.188:37597 requested a full tablet report, sending...
07:02:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.039501 10918 ts_manager.cc:194] Registered new tserver with Master: 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.040699 11451 heartbeater.cc:347] Connected to a master server at 127.10.118.189:44731
07:02:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.040838 11451 heartbeater.cc:464] Registering TS with master...
07:02:04.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.041185 11451 heartbeater.cc:511] Master 127.10.118.189:44731 requested a full tablet report, sending...
07:02:04.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.042187 10837 ts_manager.cc:194] Registered new tserver with Master: 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.044723 10714 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:02:04.046 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:02:04.046 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:02:04.047 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.118.190"
port: 40067
}
bound_http_address {
host: "127.10.118.190"
port: 43003
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.118.189"
port: 44731
}
bound_http_address {
host: "127.10.118.189"
port: 38535
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.118.188"
port: 37597
}
bound_http_address {
host: "127.10.118.188"
port: 42221
}
}
}
07:02:04.048 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:02:04.048 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.118.129"
port: 37281
}
bound_http_address {
host: "127.10.118.129"
port: 39603
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.118.130"
port: 39113
}
bound_http_address {
host: "127.10.118.130"
port: 35099
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.118.131"
port: 45339
}
bound_http_address {
host: "127.10.118.131"
port: 41399
}
}
}
07:02:04.049 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:02:04.051 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:02:04.053 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.118.188:37597(127.10.118.188:37597)] connecting to peer
07:02:04.054 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:04.055 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.118.189:44731(127.10.118.189:44731)] connecting to peer
07:02:04.055 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.118.188:37597(127.10.118.188:37597)] Successfully connected to peer
07:02:04.056 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.118.190:40067(127.10.118.190:40067)] Successfully connected to peer
07:02:04.058 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.118.188
07:02:04.058 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.118.190
07:02:04.060 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.118.189:44731(127.10.118.189:44731)] Successfully connected to peer
07:02:04.062 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.118.189
07:02:04.072 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa5b2d4e7, L:/127.0.0.1:56044 - R:/127.10.118.190:40067] using SASL/PLAIN
07:02:04.075 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:02:04.075 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=22), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: OK, deferred=null) after lookup
07:02:04.078 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfa105b05, L:/127.0.0.1:39480 - R:/127.10.118.188:37597] using SASL/PLAIN
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.077708 10756 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56044:
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.078 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfc2d2147, L:/127.0.0.1:60642 - R:/127.10.118.189:44731] using SASL/PLAIN
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:04.080834 10756 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:02:04.106 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table bbb56e8a843447919eebce98c5476584
07:02:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.145460 11243 tablet_service.cc:1467] Processing CreateTablet for tablet f95453e293a245db8f6af3c4d555f1ce (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:02:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.145447 11100 tablet_service.cc:1467] Processing CreateTablet for tablet 1d8c2da73a52449c9ef8819f302b2205 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:02:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.145601 11383 tablet_service.cc:1467] Processing CreateTablet for tablet 1d8c2da73a52449c9ef8819f302b2205 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:02:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.146402 11101 tablet_service.cc:1467] Processing CreateTablet for tablet f95453e293a245db8f6af3c4d555f1ce (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:02:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.146708 11384 tablet_service.cc:1467] Processing CreateTablet for tablet f95453e293a245db8f6af3c4d555f1ce (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:02:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.145613 11242 tablet_service.cc:1467] Processing CreateTablet for tablet 1d8c2da73a52449c9ef8819f302b2205 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:02:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.147439 11100 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d8c2da73a52449c9ef8819f302b2205. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.147456 11383 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d8c2da73a52449c9ef8819f302b2205. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.147989 11101 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f95453e293a245db8f6af3c4d555f1ce. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.148152 11242 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d8c2da73a52449c9ef8819f302b2205. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.149077 11384 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f95453e293a245db8f6af3c4d555f1ce. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.149354 11243 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f95453e293a245db8f6af3c4d555f1ce. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.169039 11475 tablet_bootstrap.cc:492] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882: Bootstrap starting.
07:02:04.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.174844 11476 tablet_bootstrap.cc:492] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: Bootstrap starting.
07:02:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.179286 11476 tablet_bootstrap.cc:654] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: Neither blocks nor log segments found. Creating new log.
07:02:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.181159 11475 tablet_bootstrap.cc:654] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882: Neither blocks nor log segments found. Creating new log.
07:02:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.181154 11476 log.cc:824] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: Log is configured to *not* fsync() on all Append() calls
07:02:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.183153 11475 log.cc:824] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882: Log is configured to *not* fsync() on all Append() calls
07:02:04.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.185377 11477 tablet_bootstrap.cc:492] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: Bootstrap starting.
07:02:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.188701 11475 tablet_bootstrap.cc:492] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882: No bootstrap required, opened a new log
07:02:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.189193 11475 ts_tablet_manager.cc:1367] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882: Time spent bootstrapping tablet: real 0.021s user 0.012s sys 0.000s
07:02:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.190416 11477 tablet_bootstrap.cc:654] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: Neither blocks nor log segments found. Creating new log.
07:02:04.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.191946 11477 log.cc:824] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: Log is configured to *not* fsync() on all Append() calls
07:02:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.196301 11476 tablet_bootstrap.cc:492] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: No bootstrap required, opened a new log
07:02:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.196983 11476 ts_tablet_manager.cc:1367] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: Time spent bootstrapping tablet: real 0.025s user 0.010s sys 0.005s
07:02:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.201968 11477 tablet_bootstrap.cc:492] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: No bootstrap required, opened a new log
07:02:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.201835 11475 raft_consensus.cc:348] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.202550 11477 ts_tablet_manager.cc:1367] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: Time spent bootstrapping tablet: real 0.018s user 0.011s sys 0.005s
07:02:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.202625 11475 raft_consensus.cc:374] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.202749 11475 raft_consensus.cc:729] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b6156414e84176a7e601627eb4e882, State: Initialized, Role: FOLLOWER
07:02:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.203819 11475 consensus_queue.cc:260] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.207626 11476 raft_consensus.cc:348] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.208448 11476 raft_consensus.cc:374] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.208663 11476 raft_consensus.cc:729] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d4ed4894d994d448aa50bfaf76ab885, State: Initialized, Role: FOLLOWER
07:02:04.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.209759 11476 consensus_queue.cc:260] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.213610 11477 raft_consensus.cc:348] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.215559 11450 heartbeater.cc:503] Master 127.10.118.190:40067 was elected leader, sending a full tablet report...
07:02:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.215617 11477 raft_consensus.cc:374] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.216876 11477 raft_consensus.cc:729] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 929a341d7e2447b0ae227d0ebec8ec81, State: Initialized, Role: FOLLOWER
07:02:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.218487 11477 consensus_queue.cc:260] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.224915 11475 ts_tablet_manager.cc:1398] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882: Time spent starting tablet: real 0.035s user 0.016s sys 0.006s
07:02:04.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.228271 11309 heartbeater.cc:503] Master 127.10.118.190:40067 was elected leader, sending a full tablet report...
07:02:04.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.229631 11477 ts_tablet_manager.cc:1398] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: Time spent starting tablet: real 0.027s user 0.016s sys 0.000s
07:02:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.233611 11476 ts_tablet_manager.cc:1398] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: Time spent starting tablet: real 0.036s user 0.016s sys 0.005s
07:02:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.234688 11475 tablet_bootstrap.cc:492] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882: Bootstrap starting.
07:02:04.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.236760 11475 tablet_bootstrap.cc:654] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882: Neither blocks nor log segments found. Creating new log.
07:02:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.237311 11476 tablet_bootstrap.cc:492] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885: Bootstrap starting.
07:02:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.239022 11476 tablet_bootstrap.cc:654] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885: Neither blocks nor log segments found. Creating new log.
07:02:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.241282 11477 tablet_bootstrap.cc:492] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81: Bootstrap starting.
07:02:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.243654 11477 tablet_bootstrap.cc:654] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81: Neither blocks nor log segments found. Creating new log.
07:02:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.245113 11475 tablet_bootstrap.cc:492] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882: No bootstrap required, opened a new log
07:02:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.245381 11475 ts_tablet_manager.cc:1367] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.007s
07:02:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.246387 11475 raft_consensus.cc:348] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.246734 11475 raft_consensus.cc:374] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.246912 11475 raft_consensus.cc:729] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b6156414e84176a7e601627eb4e882, State: Initialized, Role: FOLLOWER
07:02:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.247318 11475 consensus_queue.cc:260] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.252636 11475 ts_tablet_manager.cc:1398] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
07:02:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.258625 11476 tablet_bootstrap.cc:492] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885: No bootstrap required, opened a new log
07:02:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.258822 11476 ts_tablet_manager.cc:1367] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885: Time spent bootstrapping tablet: real 0.022s user 0.017s sys 0.001s
07:02:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.259358 11477 tablet_bootstrap.cc:492] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81: No bootstrap required, opened a new log
07:02:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.259521 11477 ts_tablet_manager.cc:1367] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81: Time spent bootstrapping tablet: real 0.018s user 0.016s sys 0.000s
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.259758 11476 raft_consensus.cc:348] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.259974 11476 raft_consensus.cc:374] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.260053 11476 raft_consensus.cc:729] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d4ed4894d994d448aa50bfaf76ab885, State: Initialized, Role: FOLLOWER
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.260329 11477 raft_consensus.cc:348] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.260378 11476 consensus_queue.cc:260] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.260537 11477 raft_consensus.cc:374] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.260769 11477 raft_consensus.cc:729] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 929a341d7e2447b0ae227d0ebec8ec81, State: Initialized, Role: FOLLOWER
07:02:04.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.261134 11477 consensus_queue.cc:260] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.264461 11477 ts_tablet_manager.cc:1398] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
07:02:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.267346 11476 ts_tablet_manager.cc:1398] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
07:02:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:04.271493 11311 tablet.cc:2369] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:02:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:04.282248 11452 tablet.cc:2369] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:02:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.433225 11481 raft_consensus.cc:482] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.433547 11481 raft_consensus.cc:504] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.435914 11481 leader_election.cc:288] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281), 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.442873 11121 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f95453e293a245db8f6af3c4d555f1ce" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d4ed4894d994d448aa50bfaf76ab885" is_pre_election: true
07:02:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.442873 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f95453e293a245db8f6af3c4d555f1ce" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81" is_pre_election: true
07:02:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.443477 11263 raft_consensus.cc:2454] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b6156414e84176a7e601627eb4e882 in term 0.
07:02:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.443473 11121 raft_consensus.cc:2454] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b6156414e84176a7e601627eb4e882 in term 0.
07:02:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.444180 11331 leader_election.cc:302] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 10b6156414e84176a7e601627eb4e882, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.444597 11481 raft_consensus.cc:2781] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.444746 11481 raft_consensus.cc:482] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.444887 11481 raft_consensus.cc:3037] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.446986 11481 raft_consensus.cc:504] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.447623 11481 leader_election.cc:288] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 election: Requested vote from peers 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281), 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:04.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.448204 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f95453e293a245db8f6af3c4d555f1ce" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:04.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.448212 11121 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f95453e293a245db8f6af3c4d555f1ce" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d4ed4894d994d448aa50bfaf76ab885"
07:02:04.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.448393 11121 raft_consensus.cc:3037] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.448383 11263 raft_consensus.cc:3037] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.450371 11263 raft_consensus.cc:2454] T f95453e293a245db8f6af3c4d555f1ce P 929a341d7e2447b0ae227d0ebec8ec81 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 10b6156414e84176a7e601627eb4e882 in term 1.
07:02:04.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.450417 11121 raft_consensus.cc:2454] T f95453e293a245db8f6af3c4d555f1ce P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 10b6156414e84176a7e601627eb4e882 in term 1.
07:02:04.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.450793 11331 leader_election.cc:302] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 10b6156414e84176a7e601627eb4e882, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.451105 11481 raft_consensus.cc:2781] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Leader election won for term 1
07:02:04.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.451498 11481 raft_consensus.cc:686] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [term 1 LEADER]: Becoming Leader. State: Replica: 10b6156414e84176a7e601627eb4e882, State: Running, Role: LEADER
07:02:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.452386 11481 consensus_queue.cc:237] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } }
07:02:04.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.458257 10756 catalog_manager.cc:5484] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 reported cstate change: term changed from 0 to 1, leader changed from <none> to 10b6156414e84176a7e601627eb4e882 (127.10.118.131). New cstate: current_term: 1 leader_uuid: "10b6156414e84176a7e601627eb4e882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } health_report { overall_health: UNKNOWN } } }
07:02:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:04.466830 11169 tablet.cc:2369] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:02:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.480170 11482 raft_consensus.cc:482] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.480381 11482 raft_consensus.cc:504] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.482369 11482 leader_election.cc:288] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113), 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:04.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.491417 11481 raft_consensus.cc:482] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:04.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.491617 11481 raft_consensus.cc:504] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.492594 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81" is_pre_election: true
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.492691 11481 leader_election.cc:288] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281), 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.492851 11263 raft_consensus.cc:2454] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 0.
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.492920 11404 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10b6156414e84176a7e601627eb4e882" is_pre_election: true
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.493271 11404 raft_consensus.cc:2454] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 0.
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.493350 11045 leader_election.cc:302] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7d4ed4894d994d448aa50bfaf76ab885, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.493824 11482 raft_consensus.cc:2781] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.493867 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81" is_pre_election: true
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.493958 11482 raft_consensus.cc:482] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.494040 11482 raft_consensus.cc:3037] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.494060 11263 raft_consensus.cc:2454] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b6156414e84176a7e601627eb4e882 in term 0.
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.494411 11331 leader_election.cc:302] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 10b6156414e84176a7e601627eb4e882, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.494695 11481 raft_consensus.cc:2781] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.494767 11481 raft_consensus.cc:482] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.494918 11481 raft_consensus.cc:3037] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.495029 11121 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d4ed4894d994d448aa50bfaf76ab885" is_pre_election: true
07:02:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.496619 11482 raft_consensus.cc:504] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.497045 11121 raft_consensus.cc:2379] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 10b6156414e84176a7e601627eb4e882 in current term 1: Already voted for candidate 7d4ed4894d994d448aa50bfaf76ab885 in this term.
07:02:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.497592 11481 raft_consensus.cc:504] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.497745 11482 leader_election.cc:288] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 election: Requested vote from peers 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113), 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.498049 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.498200 11263 raft_consensus.cc:3037] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.498828 11262 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.500049 11481 leader_election.cc:288] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 election: Requested vote from peers 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281), 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113)
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.500631 11263 raft_consensus.cc:2454] T 1d8c2da73a52449c9ef8819f302b2205 P 929a341d7e2447b0ae227d0ebec8ec81 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 1.
07:02:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.500780 11404 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10b6156414e84176a7e601627eb4e882"
07:02:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.501014 11045 leader_election.cc:302] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7d4ed4894d994d448aa50bfaf76ab885, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.501072 11404 raft_consensus.cc:2379] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 7d4ed4894d994d448aa50bfaf76ab885 in current term 1: Already voted for candidate 10b6156414e84176a7e601627eb4e882 in this term.
07:02:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.501427 11121 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d8c2da73a52449c9ef8819f302b2205" candidate_uuid: "10b6156414e84176a7e601627eb4e882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d4ed4894d994d448aa50bfaf76ab885"
07:02:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.501528 11482 raft_consensus.cc:2781] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 FOLLOWER]: Leader election won for term 1
07:02:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.502336 11482 raft_consensus.cc:686] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 LEADER]: Becoming Leader. State: Replica: 7d4ed4894d994d448aa50bfaf76ab885, State: Running, Role: LEADER
07:02:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.502936 11482 consensus_queue.cc:237] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.507486 11331 leader_election.cc:302] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 10b6156414e84176a7e601627eb4e882; no voters: 7d4ed4894d994d448aa50bfaf76ab885, 929a341d7e2447b0ae227d0ebec8ec81
07:02:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.508121 11481 raft_consensus.cc:2726] T 1d8c2da73a52449c9ef8819f302b2205 P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
07:02:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.508091 10756 catalog_manager.cc:5484] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129). New cstate: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } health_report { overall_health: UNKNOWN } } }
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.901965 10756 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56056:
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:02:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:04.903102 10756 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:02:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.916563 11101 tablet_service.cc:1467] Processing CreateTablet for tablet 9226093346b347578a8fd0d06793f73e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fb4a33d4c2ba4267b38b9b0977443ac0]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:02:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.917022 11242 tablet_service.cc:1467] Processing CreateTablet for tablet 9226093346b347578a8fd0d06793f73e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fb4a33d4c2ba4267b38b9b0977443ac0]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:02:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.917877 11242 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9226093346b347578a8fd0d06793f73e. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.917877 11101 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9226093346b347578a8fd0d06793f73e. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.923878 11383 tablet_service.cc:1467] Processing CreateTablet for tablet 9226093346b347578a8fd0d06793f73e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fb4a33d4c2ba4267b38b9b0977443ac0]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:02:04.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.925277 11383 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9226093346b347578a8fd0d06793f73e. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:04.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.926199 11504 tablet_bootstrap.cc:492] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: Bootstrap starting.
07:02:04.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.927729 11504 tablet_bootstrap.cc:654] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: Neither blocks nor log segments found. Creating new log.
07:02:04.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.929805 11504 tablet_bootstrap.cc:492] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: No bootstrap required, opened a new log
07:02:04.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.930197 11504 ts_tablet_manager.cc:1367] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.004s
07:02:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.931231 11504 raft_consensus.cc:348] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.931777 11506 tablet_bootstrap.cc:492] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: Bootstrap starting.
07:02:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.932111 11504 raft_consensus.cc:374] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.932224 11504 raft_consensus.cc:729] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 929a341d7e2447b0ae227d0ebec8ec81, State: Initialized, Role: FOLLOWER
07:02:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.932559 11504 consensus_queue.cc:260] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.932902 11507 tablet_bootstrap.cc:492] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: Bootstrap starting.
07:02:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.933729 11506 tablet_bootstrap.cc:654] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: Neither blocks nor log segments found. Creating new log.
07:02:04.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.935635 11507 tablet_bootstrap.cc:654] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: Neither blocks nor log segments found. Creating new log.
07:02:04.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.936017 11506 tablet_bootstrap.cc:492] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: No bootstrap required, opened a new log
07:02:04.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.936210 11506 ts_tablet_manager.cc:1367] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
07:02:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.937167 11506 raft_consensus.cc:348] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.937386 11506 raft_consensus.cc:374] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.937484 11506 raft_consensus.cc:729] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d4ed4894d994d448aa50bfaf76ab885, State: Initialized, Role: FOLLOWER
07:02:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.937769 11506 consensus_queue.cc:260] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.938498 11504 ts_tablet_manager.cc:1398] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: Time spent starting tablet: real 0.008s user 0.007s sys 0.000s
07:02:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.938412 11482 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.938683 11482 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:04.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.938952 11507 tablet_bootstrap.cc:492] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: No bootstrap required, opened a new log
07:02:04.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.939229 11507 ts_tablet_manager.cc:1367] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: Time spent bootstrapping tablet: real 0.006s user 0.005s sys 0.000s
07:02:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.940344 11507 raft_consensus.cc:348] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.940706 11507 raft_consensus.cc:374] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.941628 11507 raft_consensus.cc:729] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b6156414e84176a7e601627eb4e882, State: Initialized, Role: FOLLOWER
07:02:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.942052 11507 consensus_queue.cc:260] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.942770 11507 ts_tablet_manager.cc:1398] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
07:02:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.943102 11481 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.943419 11481 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.943356 11508 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.943727 11508 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.943758 11506 ts_tablet_manager.cc:1398] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: Time spent starting tablet: real 0.007s user 0.000s sys 0.003s
07:02:04.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.973985 11481 consensus_queue.cc:1035] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:04.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.974675 11482 raft_consensus.cc:482] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:04.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.974907 11482 raft_consensus.cc:504] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.975636 11482 leader_election.cc:288] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113), 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.976596 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9226093346b347578a8fd0d06793f73e" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81" is_pre_election: true
07:02:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.976902 11263 raft_consensus.cc:2454] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 0.
07:02:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.977421 11404 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9226093346b347578a8fd0d06793f73e" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10b6156414e84176a7e601627eb4e882" is_pre_election: true
07:02:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.977525 11045 leader_election.cc:302] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7d4ed4894d994d448aa50bfaf76ab885, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.977659 11404 raft_consensus.cc:2454] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 0.
07:02:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.977878 11482 raft_consensus.cc:2781] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.977974 11482 raft_consensus.cc:482] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.978092 11482 raft_consensus.cc:3037] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.978595 11491 consensus_queue.cc:1035] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.981141 11482 raft_consensus.cc:504] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:04.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.982549 11263 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9226093346b347578a8fd0d06793f73e" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "929a341d7e2447b0ae227d0ebec8ec81"
07:02:04.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.982832 11263 raft_consensus.cc:3037] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.987605 11263 raft_consensus.cc:2454] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 1.
07:02:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.988899 11403 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9226093346b347578a8fd0d06793f73e" candidate_uuid: "7d4ed4894d994d448aa50bfaf76ab885" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10b6156414e84176a7e601627eb4e882"
07:02:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.989223 11403 raft_consensus.cc:3037] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 0 FOLLOWER]: Advancing to term 1
07:02:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.993554 11403 raft_consensus.cc:2454] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7d4ed4894d994d448aa50bfaf76ab885 in term 1.
07:02:04.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.994524 11045 leader_election.cc:302] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7d4ed4894d994d448aa50bfaf76ab885, 929a341d7e2447b0ae227d0ebec8ec81; no voters:
07:02:04.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.994997 11491 raft_consensus.cc:2781] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 FOLLOWER]: Leader election won for term 1
07:02:04.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.996927 11482 leader_election.cc:288] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [CANDIDATE]: Term 1 election: Requested vote from peers 929a341d7e2447b0ae227d0ebec8ec81 (127.10.118.130:39113), 10b6156414e84176a7e601627eb4e882 (127.10.118.131:45339)
07:02:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.999218 11491 raft_consensus.cc:686] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [term 1 LEADER]: Becoming Leader. State: Replica: 7d4ed4894d994d448aa50bfaf76ab885, State: Running, Role: LEADER
07:02:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:04.999564 11491 consensus_queue.cc:237] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } }
07:02:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.001967 11482 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: TxnStatusTablet state changed. Reason: New leader 7d4ed4894d994d448aa50bfaf76ab885. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.002686 11512 consensus_queue.cc:1035] T f95453e293a245db8f6af3c4d555f1ce P 10b6156414e84176a7e601627eb4e882 [LEADER]: Connected to new peer: Peer: permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.005170 11482 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: This TxnStatusTablet replica's current role is: LEADER
07:02:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.016125 11491 consensus_queue.cc:1035] T 1d8c2da73a52449c9ef8819f302b2205 P 7d4ed4894d994d448aa50bfaf76ab885 [LEADER]: Connected to new peer: Peer: permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
07:02:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.022969 11523 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
07:02:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.023449 11523 txn_status_manager.cc:929] Loading transaction status metadata into memory...
07:02:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.035342 10756 catalog_manager.cc:5484] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129). New cstate: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } health_report { overall_health: UNKNOWN } } }
07:02:05.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.109035 11404 raft_consensus.cc:1261] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882 [term 1 FOLLOWER]: Refusing update from remote peer 7d4ed4894d994d448aa50bfaf76ab885: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.110193 11516 consensus_queue.cc:1035] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.110474 11263 raft_consensus.cc:1261] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81 [term 1 FOLLOWER]: Refusing update from remote peer 7d4ed4894d994d448aa50bfaf76ab885: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:05.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.111164 11482 consensus_queue.cc:1035] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885 [LEADER]: Connected to new peer: Peer: permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.123975 11508 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: TxnStatusTablet state changed. Reason: New leader 7d4ed4894d994d448aa50bfaf76ab885. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.124415 11508 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.135387 11517 mvcc.cc:205] Tried to move back new op lower bound from 7021014733234212864 to 7021014732801007616. Current Snapshot: MvccSnapshot[applied={T|T < 7021014733234212864}]
07:02:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.139746 11512 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: TxnStatusTablet state changed. Reason: New leader 7d4ed4894d994d448aa50bfaf76ab885. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.140106 11512 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:05.149 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
07:02:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.148520 11521 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.148877 11521 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: This TxnStatusTablet replica's current role is: LEADER
07:02:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.148707 11512 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.150352 11508 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.150594 11508 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 929a341d7e2447b0ae227d0ebec8ec81: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:05.157 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table bbb56e8a843447919eebce98c5476584
07:02:05.157 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.118.190:40067
07:02:05.158 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:02:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.158284 11521 tablet_replica.cc:439] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7d4ed4894d994d448aa50bfaf76ab885" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d4ed4894d994d448aa50bfaf76ab885" member_type: VOTER last_known_addr { host: "127.10.118.129" port: 37281 } } peers { permanent_uuid: "929a341d7e2447b0ae227d0ebec8ec81" member_type: VOTER last_known_addr { host: "127.10.118.130" port: 39113 } } peers { permanent_uuid: "10b6156414e84176a7e601627eb4e882" member_type: VOTER last_known_addr { host: "127.10.118.131" port: 45339 } } }
07:02:05.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.158567 11521 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 7d4ed4894d994d448aa50bfaf76ab885: This TxnStatusTablet replica's current role is: LEADER
07:02:05.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.158816 11512 tablet_replica.cc:442] T 9226093346b347578a8fd0d06793f73e P 10b6156414e84176a7e601627eb4e882: This TxnStatusTablet replica's current role is: FOLLOWER
07:02:05.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.158941 10714 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 10725
07:02:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.163053 10796 generic_service.cc:196] Checking for leaks (request via RPC)
07:02:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.308296 10714 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 10725
07:02:05.339 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.118.190:40067(127.10.118.190:40067)] handling channelInactive
07:02:05.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state READY due to: connection closed
07:02:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.340356 10903 connection.cc:535] server connection from 127.0.0.1:45446 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:02:05.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:02:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.595192 11450 heartbeater.cc:644] Failed to heartbeat to 127.10.118.190:40067 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.606124 11167 heartbeater.cc:644] Failed to heartbeat to 127.10.118.190:40067 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.608719 11538 raft_consensus.cc:482] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Starting pre-election (detected failure of leader df97c70ed96443ac833c3a62b3385d37)
07:02:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.608721 11539 raft_consensus.cc:482] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader df97c70ed96443ac833c3a62b3385d37)
07:02:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.608915 11539 raft_consensus.cc:504] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.608932 11538 raft_consensus.cc:504] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.610283 11539 leader_election.cc:288] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067), 446ba19691fd4c469d75f3459ec45ccd (127.10.118.189:44731)
07:02:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.610333 11538 leader_election.cc:288] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067), feca55a551e141dd8fff0d68430dcc15 (127.10.118.188:37597)
07:02:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.611470 10928 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "446ba19691fd4c469d75f3459ec45ccd" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "feca55a551e141dd8fff0d68430dcc15" is_pre_election: true
07:02:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.611485 10846 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "feca55a551e141dd8fff0d68430dcc15" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "446ba19691fd4c469d75f3459ec45ccd" is_pre_election: true
07:02:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.611758 10928 raft_consensus.cc:2454] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 446ba19691fd4c469d75f3459ec45ccd in term 1.
07:02:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.611828 10846 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate feca55a551e141dd8fff0d68430dcc15 in term 1.
07:02:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.612190 10903 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.612673 10822 leader_election.cc:334] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.613127 10903 leader_election.cc:302] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 446ba19691fd4c469d75f3459ec45ccd, feca55a551e141dd8fff0d68430dcc15; no voters:
07:02:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.613077 10824 leader_election.cc:302] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 446ba19691fd4c469d75f3459ec45ccd, feca55a551e141dd8fff0d68430dcc15; no voters: df97c70ed96443ac833c3a62b3385d37
07:02:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.613528 11539 raft_consensus.cc:2781] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Leader pre-election won for term 2
07:02:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.613655 11539 raft_consensus.cc:482] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Starting leader election (detected failure of leader df97c70ed96443ac833c3a62b3385d37)
07:02:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.613788 11539 raft_consensus.cc:3037] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 1 FOLLOWER]: Advancing to term 2
07:02:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.614068 10903 leader_election.cc:334] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.615404 11538 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Leader pre-election won for term 2
07:02:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.615530 11538 raft_consensus.cc:482] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Starting leader election (detected failure of leader df97c70ed96443ac833c3a62b3385d37)
07:02:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.615634 11538 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 1 FOLLOWER]: Advancing to term 2
07:02:05.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.616297 11539 raft_consensus.cc:504] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.617113 11539 leader_election.cc:288] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [CANDIDATE]: Term 2 election: Requested vote from peers df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067), 446ba19691fd4c469d75f3459ec45ccd (127.10.118.189:44731)
07:02:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.617893 10846 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "feca55a551e141dd8fff0d68430dcc15" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "446ba19691fd4c469d75f3459ec45ccd"
07:02:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.618438 10903 leader_election.cc:334] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.618394 11538 raft_consensus.cc:504] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.618871 10846 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate feca55a551e141dd8fff0d68430dcc15 in current term 2: Already voted for candidate 446ba19691fd4c469d75f3459ec45ccd in this term.
07:02:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.619167 11538 leader_election.cc:288] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 2 election: Requested vote from peers df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067), feca55a551e141dd8fff0d68430dcc15 (127.10.118.188:37597)
07:02:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.619395 10903 leader_election.cc:302] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: feca55a551e141dd8fff0d68430dcc15; no voters: 446ba19691fd4c469d75f3459ec45ccd, df97c70ed96443ac833c3a62b3385d37
07:02:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.619737 11539 raft_consensus.cc:2726] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
07:02:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.620218 10928 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "446ba19691fd4c469d75f3459ec45ccd" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "feca55a551e141dd8fff0d68430dcc15"
07:02:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.620352 10822 leader_election.cc:334] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.620476 10928 raft_consensus.cc:2379] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 446ba19691fd4c469d75f3459ec45ccd in current term 2: Already voted for candidate feca55a551e141dd8fff0d68430dcc15 in this term.
07:02:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.620954 10824 leader_election.cc:302] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 446ba19691fd4c469d75f3459ec45ccd; no voters: df97c70ed96443ac833c3a62b3385d37, feca55a551e141dd8fff0d68430dcc15
07:02:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.621246 11538 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
07:02:05.654 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.655 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.655 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.655 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.10.118.190:40067(127.10.118.190:40067) for tablet Kudu Master: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.674 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.675 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.675 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.675 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.676 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.694 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.695 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.696 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.714 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.716 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.716 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.716 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.717 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.734 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.735 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.735 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.735 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.736 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.774 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.775 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.776 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.776 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.814 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.815 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.815 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.815 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.816 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.817 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] sending RPC to server master-127.10.118.190:40067, [2ms] received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [163ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, deferred=Deferred@736381888(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/255253488@e75799a -> (continuation of Deferred@2073672915 after retry RPC after error@1604211184) -> (continuation of Deferred@1454226312 after retry RPC after error@1932632430) -> (continuation of Deferred@1173943551 after retry RPC after error@1757686881) -> (continuation of Deferred@1401862922 after retry RPC after error@759843524) -> (continuation of Deferred@790043806 after retry RPC after error@1719245369), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/851091760@40893b17 -> (continuation of Deferred@2073672915 after retry RPC after error@1604211184) -> (continuation of Deferred@1454226312 after retry RPC after error@1932632430) -> (continuation of Deferred@1173943551 after retry RPC after error@1757686881) -> (continuation of Deferred@1401862922 after retry RPC after error@759843524) -> (continuation of Deferred@790043806 after retry RPC after error@1719245369)))
07:02:05.817 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] sending RPC to server master-127.10.118.190:40067, [2ms] received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [163ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, deferred=Deferred@736381888(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/255253488@e75799a -> (continuation of Deferred@2073672915 after retry RPC after error@1604211184) -> (continuation of Deferred@1454226312 after retry RPC after error@1932632430) -> (continuation of Deferred@1173943551 after retry RPC after error@1757686881) -> (continuation of Deferred@1401862922 after retry RPC after error@759843524) -> (continuation of Deferred@790043806 after retry RPC after error@1719245369), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/851091760@40893b17 -> (continuation of Deferred@2073672915 after retry RPC after error@1604211184) -> (continuation of Deferred@1454226312 after retry RPC after error@1932632430) -> (continuation of Deferred@1173943551 after retry RPC after error@1757686881) -> (continuation of Deferred@1401862922 after retry RPC after error@759843524) -> (continuation of Deferred@790043806 after retry RPC after error@1719245369)))
07:02:05.834 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.835 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.835 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.836 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.837 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.854 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.856 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.856 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.856 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.856 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.874 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.875 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.876 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.876 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.877 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.894 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.896 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.913 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.914 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.914 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.915 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.934 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.936 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.936 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.936 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.936 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.956966 11538 raft_consensus.cc:482] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.957170 11538 raft_consensus.cc:504] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.957747 11538 leader_election.cc:288] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067), feca55a551e141dd8fff0d68430dcc15 (127.10.118.188:37597)
07:02:05.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.958317 10928 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "446ba19691fd4c469d75f3459ec45ccd" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "feca55a551e141dd8fff0d68430dcc15" is_pre_election: true
07:02:05.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.958505 10928 raft_consensus.cc:2454] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 446ba19691fd4c469d75f3459ec45ccd in term 2.
07:02:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.958904 10824 leader_election.cc:302] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 446ba19691fd4c469d75f3459ec45ccd, feca55a551e141dd8fff0d68430dcc15; no voters:
07:02:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.959129 10822 leader_election.cc:334] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.959255 11538 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Leader pre-election won for term 3
07:02:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.959336 11538 raft_consensus.cc:482] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.959447 11538 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 2 FOLLOWER]: Advancing to term 3
07:02:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.961279 11538 raft_consensus.cc:504] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.961817 11538 leader_election.cc:288] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 3 election: Requested vote from peers df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067), feca55a551e141dd8fff0d68430dcc15 (127.10.118.188:37597)
07:02:05.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.962405 10928 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "446ba19691fd4c469d75f3459ec45ccd" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "feca55a551e141dd8fff0d68430dcc15"
07:02:05.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.962620 10928 raft_consensus.cc:3037] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 2 FOLLOWER]: Advancing to term 3
07:02:05.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:05.963080 10822 leader_election.cc:334] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.964754 10928 raft_consensus.cc:2454] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 446ba19691fd4c469d75f3459ec45ccd in term 3.
07:02:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.965169 10824 leader_election.cc:302] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 446ba19691fd4c469d75f3459ec45ccd, feca55a551e141dd8fff0d68430dcc15; no voters: df97c70ed96443ac833c3a62b3385d37
07:02:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.965492 11538 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 3 FOLLOWER]: Leader election won for term 3
07:02:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.965970 11538 raft_consensus.cc:686] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [term 3 LEADER]: Becoming Leader. State: Replica: 446ba19691fd4c469d75f3459ec45ccd, State: Running, Role: LEADER
07:02:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.966473 11538 consensus_queue.cc:237] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } }
07:02:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.967979 11544 sys_catalog.cc:455] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 446ba19691fd4c469d75f3459ec45ccd. Latest consensus state: current_term: 3 leader_uuid: "446ba19691fd4c469d75f3459ec45ccd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.968221 11544 sys_catalog.cc:458] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: This master's current role is: LEADER
07:02:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.968858 11545 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:02:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.972196 11545 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584]
07:02:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.972774 11545 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=fb4a33d4c2ba4267b38b9b0977443ac0]
07:02:05.973 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.975466 11545 tablet_loader.cc:96] loaded metadata for tablet 1d8c2da73a52449c9ef8819f302b2205 (table txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584])
07:02:05.975 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.975 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.975939 11545 tablet_loader.cc:96] loaded metadata for tablet 9226093346b347578a8fd0d06793f73e (table kudu_system.kudu_transactions [id=fb4a33d4c2ba4267b38b9b0977443ac0])
07:02:05.976 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:05.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.976336 11545 tablet_loader.cc:96] loaded metadata for tablet f95453e293a245db8f6af3c4d555f1ce (table txn_manager_fallback_rolling [id=bbb56e8a843447919eebce98c5476584])
07:02:05.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.976696 11545 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:02:05.977 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731; will retry
07:02:05.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.977756 11545 catalog_manager.cc:1259] Loaded cluster ID: 1ccf4cff07bf472c83291437d4679755
07:02:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.977939 11545 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:02:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.979276 11545 catalog_manager.cc:1504] Loading token signing keys...
07:02:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.980193 11545 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd: Loaded TSK: 0
07:02:05.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:05.981448 11545 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:02:06.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.067216 10928 raft_consensus.cc:1261] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [term 3 FOLLOWER]: Refusing update from remote peer 446ba19691fd4c469d75f3459ec45ccd: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
07:02:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.067919 11544 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [LEADER]: Connected to new peer: Peer: permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
07:02:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.069617 11539 sys_catalog.cc:455] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 446ba19691fd4c469d75f3459ec45ccd. Latest consensus state: current_term: 3 leader_uuid: "446ba19691fd4c469d75f3459ec45ccd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.069875 11539 sys_catalog.cc:458] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: This master's current role is: FOLLOWER
07:02:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.071650 11538 sys_catalog.cc:455] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "446ba19691fd4c469d75f3459ec45ccd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.071985 11538 sys_catalog.cc:458] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd [sys.catalog]: This master's current role is: LEADER
07:02:06.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:06.073179 10822 consensus_peers.cc:485] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd -> Peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Couldn't send request to peer df97c70ed96443ac833c3a62b3385d37. Status: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:02:06.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.073109 11539 sys_catalog.cc:455] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "446ba19691fd4c469d75f3459ec45ccd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "df97c70ed96443ac833c3a62b3385d37" member_type: VOTER last_known_addr { host: "127.10.118.190" port: 40067 } } peers { permanent_uuid: "446ba19691fd4c469d75f3459ec45ccd" member_type: VOTER last_known_addr { host: "127.10.118.189" port: 44731 } } peers { permanent_uuid: "feca55a551e141dd8fff0d68430dcc15" member_type: VOTER last_known_addr { host: "127.10.118.188" port: 37597 } } }
07:02:06.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.073348 11539 sys_catalog.cc:458] T 00000000000000000000000000000000 P feca55a551e141dd8fff0d68430dcc15 [sys.catalog]: This master's current role is: FOLLOWER
07:02:06.094 [DEBUG - pool-48-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [24ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [144ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.10.118.188:37597,127.10.118.190:40067,127.10.118.189:44731) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, deferred=Deferred@289443335(state=PENDING, result=null, callback=(continuation of Deferred@1665490176 after retry RPC after error@390410250) -> (continuation of Deferred@1603755517 after retry RPC after error@1253232830) -> (continuation of Deferred@55978421 after retry RPC after error@1026395317) -> (continuation of Deferred@1487951448 after retry RPC after error@73091792) -> (continuation of Deferred@921878639 after retry RPC after error@1735109152) -> (continuation of Deferred@1035149827 after retry RPC after error@1675682479) -> (continuation of Deferred@1135716100 after retry RPC after error@2033266680), errback=(continuation of Deferred@1665490176 after retry RPC after error@390410250) -> (continuation of Deferred@1603755517 after retry RPC after error@1253232830) -> (continuation of Deferred@55978421 after retry RPC after error@1026395317) -> (continuation of Deferred@1487951448 after retry RPC after error@73091792) -> (continuation of Deferred@921878639 after retry RPC after error@1735109152) -> (continuation of Deferred@1035149827 after retry RPC after error@1675682479) -> (continuation of Deferred@1135716100 after retry RPC after error@2033266680)))
07:02:06.095 [WARN - pool-48-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:748)
07:02:06.114 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.10.118.190:40067(127.10.118.190:40067)] connecting to peer
07:02:06.115 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:06.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.118.190:40067(127.10.118.190:40067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:06.115 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.118.190:40067: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067
07:02:06.118 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:02:06.118 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.188:37597, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.190:40067, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.118.189:44731, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.190:40067: Network error: Failed to connect to peer master-127.10.118.190:40067(127.10.118.190:40067): Connection refused: /127.10.118.190:40067, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.188:37597: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.10.118.189:44731: OK, deferred=null) after lookup
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.159204 10836 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60652:
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:02:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:06.167960 11309 heartbeater.cc:644] Failed to heartbeat to 127.10.118.190:40067 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111)
07:02:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:06.169585 10836 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:02:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:06.173242 11048 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.011s from last keepalive heartbeat (effective timeout is 1.000s)
07:02:06.194 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 7d4ed4894d994d448aa50bfaf76ab885 (127.10.118.129:37281): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1714114925 last_transition_timestamp: 1714114926
07:02:06.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:06.577555 10822 consensus_peers.cc:485] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd -> Peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Couldn't send request to peer df97c70ed96443ac833c3a62b3385d37. Status: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
07:02:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:06.665046 10822 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111) [suppressed 14 similar messages]
07:02:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:07.057318 10822 consensus_peers.cc:485] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd -> Peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Couldn't send request to peer df97c70ed96443ac833c3a62b3385d37. Status: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
07:02:07.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.131723 10884 master_runner.cc:186] Error getting master registration for 127.10.118.190:40067: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111),
07:02:07.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.131922 10884 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:02:07.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.169564 11310 heartbeater.cc:503] Master 127.10.118.189:44731 was elected leader, sending a full tablet report...
07:02:07.345 [DEBUG - main] (Connection.java:575) [peer master-127.10.118.188:37597(127.10.118.188:37597)] disconnecting while in state READY
07:02:07.345 [DEBUG - main] (Connection.java:575) [peer master-127.10.118.189:44731(127.10.118.189:44731)] disconnecting while in state READY
07:02:07.346 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.118.188:37597(127.10.118.188:37597)] handling channelInactive
07:02:07.347 [DEBUG - main] (Connection.java:575) [peer master-127.10.118.190:40067(127.10.118.190:40067)] disconnecting while in state TERMINATED
07:02:07.347 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.188:37597(127.10.118.188:37597)] cleaning up while in state READY due to: connection closed
07:02:07.348 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:02:07.349 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.118.189:44731(127.10.118.189:44731)] handling channelInactive
07:02:07.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.118.189:44731(127.10.118.189:44731)] cleaning up while in state READY due to: connection closed
07:02:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.349438 10714 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 11010
07:02:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.357436 11161 generic_service.cc:196] Checking for leaks (request via RPC)
07:02:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.484467 10714 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 11010
07:02:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.511082 10714 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 11171
07:02:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.517392 11303 generic_service.cc:196] Checking for leaks (request via RPC)
07:02:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:07.542573 10822 consensus_peers.cc:485] T 00000000000000000000000000000000 P 446ba19691fd4c469d75f3459ec45ccd -> Peer df97c70ed96443ac833c3a62b3385d37 (127.10.118.190:40067): Couldn't send request to peer df97c70ed96443ac833c3a62b3385d37. Status: Network error: Client connection negotiation failed: client connection to 127.10.118.190:40067: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
07:02:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.597437 11451 heartbeater.cc:503] Master 127.10.118.189:44731 was elected leader, sending a full tablet report...
07:02:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.605262 10714 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 11171
07:02:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.622002 10714 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 11313
07:02:07.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.628880 11444 generic_service.cc:196] Checking for leaks (request via RPC)
07:02:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.718719 10714 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 11313
07:02:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.732393 10714 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 10803
07:02:07.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.733500 10877 generic_service.cc:196] Checking for leaks (request via RPC)
07:02:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.842483 10714 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 10803
07:02:07.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:07.862159 10903 connection.cc:535] server connection from 127.0.0.1:45456 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:02:07.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.862620 10714 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 10884
07:02:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.863762 10958 generic_service.cc:196] Checking for leaks (request via RPC)
07:02:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:07.970098 10714 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 10884
07:02:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:02:08Z chronyd exiting
07:02:08.119 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)