14:20:00.188 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:20:00.188 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskdRM744/test-tmp
14:20:00.188 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:20:00.188 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskdRM744/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:20:00.189 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:20:00.515 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:20:00.515 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:20:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.518352 10138 mini_oidc.cc:152] Starting JWKS server
14:20:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.521890 10138 webserver.cc:460] Webserver started at http://127.0.0.1:39953/ using document root <none> and password file <none>
14:20:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.522397 10138 mini_oidc.cc:189] Starting OIDC Discovery server
14:20:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.524258 10138 webserver.cc:460] Webserver started at http://127.0.0.1:45009/ using document root <none> and password file <none>
14:20:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T14:20:00Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
14:20:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T14:20:00Z Disabled control of system clock
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.587569 10138 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/logs
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data/info.pb
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.190:42571
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.190
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39953/jwks.json
14:20:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:20:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:20:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.883476 10150 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.884239 10150 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.884954 10150 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.915802 10150 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.916098 10150 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:20:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.916340 10150 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.916571 10150 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.916816 10150 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:00.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.917021 10150 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:00.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.917165 10150 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39953/jwks.json
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.951649 10150 master_runner.cc:386] Master server non-default flags:
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.190:42571
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39953/jwks.json
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data/info.pb
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.190
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/logs
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 13:43:20 UTC on 1a15035a6830
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 76
14:20:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:00.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.952894 10150 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.954511 10150 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.971789 10156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:02.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.972433 10159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:00.972863 10157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:00.973424 10150 server_base.cc:1034] running on GCE node
14:20:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.076073 10150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:02.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.078755 10150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:02.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.080127 10150 hybrid_clock.cc:648] HybridClock initialized: now 1714573202080081 us; error 53 us; skew 500 ppm
14:20:02.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.081018 10150 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.087798 10150 webserver.cc:460] Webserver started at http://127.9.230.190:45901/ using document root <none> and password file <none>
14:20:02.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.088812 10150 fs_manager.cc:359] Metadata directory not provided
14:20:02.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.089035 10150 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:02.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.089533 10150 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:20:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.094477 10150 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data/instance:
14:20:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "425008e39c9541d182ddf94d18376ec0"
14:20:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:02 on dist-test-slave-kwk7"
14:20:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.095608 10150 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal/instance:
14:20:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "425008e39c9541d182ddf94d18376ec0"
14:20:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:02 on dist-test-slave-kwk7"
14:20:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.103299 10150 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
14:20:02.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.109017 10166 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:02.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.110040 10150 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:20:02.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.110368 10150 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data,/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal
14:20:02.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "425008e39c9541d182ddf94d18376ec0"
14:20:02.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:02 on dist-test-slave-kwk7"
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.110711 10150 fs_report.cc:383] FS layout report
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data/data
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:02.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.169281 10150 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:02.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.170782 10150 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:02.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.171258 10150 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.251282 10150 rpc_server.cc:307] RPC server started. Bound to: 127.9.230.190:42571
14:20:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.251348 10227 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.230.190:42571 every 8 connection(s)
14:20:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.253916 10150 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/data/info.pb
14:20:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.259199 10228 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.261917 10138 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdRM744/build/tsan/bin/kudu as pid 10150
14:20:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.262298 10138 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-0/wal/instance
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.266147 10138 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/logs
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data/info.pb
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.189:41409
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.189
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39953/jwks.json
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:20:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:20:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.270916 10228 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } has no permanent_uuid. Determining permanent_uuid...
14:20:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.298161 10228 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } has no permanent_uuid. Determining permanent_uuid...
14:20:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.300609 10170 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.304618 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:02.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.354133 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 1
14:20:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.357870 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.426230 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 2
14:20:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.430702 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:02.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.522186 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 3
14:20:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.526314 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.575258 10230 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.575825 10230 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.576303 10230 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.606537 10230 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.606835 10230 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:20:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.607000 10230 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.607200 10230 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.607403 10230 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.607589 10230 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.607731 10230 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39953/jwks.json
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.642634 10230 master_runner.cc:386] Master server non-default flags:
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.189:41409
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39953/jwks.json
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data/info.pb
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.189
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/logs
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 13:43:20 UTC on 1a15035a6830
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 76
14:20:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.643832 10230 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.645639 10230 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.660020 10239 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.669721 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 4
14:20:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.673384 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:02.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.972913 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 5
14:20:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.978701 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.526144 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 6
14:20:03.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:03.533519 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.189:41409: Network error: Client connection negotiation failed: client connection to 127.9.230.189:41409: connect: Connection refused (error 111)
14:20:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.660506 10240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:02.662173 10242 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:02.662901 10230 server_base.cc:1034] running on GCE node
14:20:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.766654 10230 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:03.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.769716 10230 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.771152 10230 hybrid_clock.cc:648] HybridClock initialized: now 1714573203771111 us; error 59 us; skew 500 ppm
14:20:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.771917 10230 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.778607 10230 webserver.cc:460] Webserver started at http://127.9.230.189:38337/ using document root <none> and password file <none>
14:20:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.779446 10230 fs_manager.cc:359] Metadata directory not provided
14:20:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.779628 10230 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.780007 10230 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:20:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.784745 10230 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data/instance:
14:20:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:03 on dist-test-slave-kwk7"
14:20:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.785790 10230 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal/instance:
14:20:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:03 on dist-test-slave-kwk7"
14:20:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.792819 10230 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
14:20:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.798483 10251 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.799594 10230 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.799918 10230 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data,/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:03 on dist-test-slave-kwk7"
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.800220 10230 fs_report.cc:383] FS layout report
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data/data
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:03.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.862941 10230 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.864383 10230 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.864841 10230 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:03.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.945998 10230 rpc_server.cc:307] RPC server started. Bound to: 127.9.230.189:41409
14:20:03.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.946091 10312 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.230.189:41409 every 8 connection(s)
14:20:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.948555 10230 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/data/info.pb
14:20:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.949378 10138 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdRM744/build/tsan/bin/kudu as pid 10230
14:20:03.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.949800 10138 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-1/wal/instance
14:20:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.952369 10138 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal
14:20:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data
14:20:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/logs
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data/info.pb
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.188:32935
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.188
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39953/jwks.json
14:20:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:20:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:20:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.955134 10313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:03.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.965869 10313 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } has no permanent_uuid. Determining permanent_uuid...
14:20:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:03.986438 10313 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } has no permanent_uuid. Determining permanent_uuid...
14:20:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.000289 10313 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } has no permanent_uuid. Determining permanent_uuid...
14:20:04.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.002766 10252 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.006747 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.056286 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 1
14:20:04.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.060453 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.128896 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 2
14:20:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.132983 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.224470 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 3
14:20:04.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.229125 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.262151 10315 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.262933 10315 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.263765 10315 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.297562 10315 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.297847 10315 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:20:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.298015 10315 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.298208 10315 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.298420 10315 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.298682 10315 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.298830 10315 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39953/jwks.json
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.333205 10315 master_runner.cc:386] Master server non-default flags:
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.188:32935
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39953/jwks.json
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data/info.pb
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.188
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/logs
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 13:43:20 UTC on 1a15035a6830
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 76
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.334595 10315 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:04.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.336159 10315 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.351122 10326 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.372741 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 4
14:20:04.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.377631 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.594180 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } attempt: 7
14:20:04.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.606680 10228 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } has no permanent_uuid. Determining permanent_uuid...
14:20:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.613049 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.671437 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 1
14:20:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.675853 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.676975 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 5
14:20:04.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.681732 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.757239 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 2
14:20:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.761624 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.846968 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 3
14:20:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.851533 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.992003 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 4
14:20:04.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.996721 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.229102 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 6
14:20:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:05.236001 10313 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.280104 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 5
14:20:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:05.284590 10228 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.230.188:32935: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.352222 10325 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:04.351768 10328 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:04.352680 10315 server_base.cc:1034] running on GCE node
14:20:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.479115 10315 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:05.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.481743 10315 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.483103 10315 hybrid_clock.cc:648] HybridClock initialized: now 1714573205483061 us; error 63 us; skew 500 ppm
14:20:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.483952 10315 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.490592 10315 webserver.cc:460] Webserver started at http://127.9.230.188:37275/ using document root <none> and password file <none>
14:20:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.491458 10315 fs_manager.cc:359] Metadata directory not provided
14:20:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.491647 10315 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:05.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.492043 10315 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:20:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.496913 10315 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data/instance:
14:20:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b"
14:20:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:05 on dist-test-slave-kwk7"
14:20:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.497972 10315 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal/instance:
14:20:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b"
14:20:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:05 on dist-test-slave-kwk7"
14:20:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.504920 10315 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
14:20:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.510349 10340 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.511413 10315 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:20:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.511705 10315 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data,/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b"
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:05 on dist-test-slave-kwk7"
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.512007 10315 fs_report.cc:383] FS layout report
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data/data
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.563063 10315 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.564507 10315 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.564934 10315 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.646757 10315 rpc_server.cc:307] RPC server started. Bound to: 127.9.230.188:32935
14:20:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.646844 10401 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.230.188:32935 every 8 connection(s)
14:20:05.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.649444 10315 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/data/info.pb
14:20:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.654601 10402 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:05.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.657508 10138 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdRM744/build/tsan/bin/kudu as pid 10315
14:20:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.657955 10138 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/master-2/wal/instance
14:20:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.669055 10402 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } has no permanent_uuid. Determining permanent_uuid...
14:20:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.699820 10402 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } has no permanent_uuid. Determining permanent_uuid...
14:20:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.709931 10402 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } has no permanent_uuid. Determining permanent_uuid...
14:20:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.726102 10402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: Bootstrap starting.
14:20:05.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.731424 10402 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: Neither blocks nor log segments found. Creating new log.
14:20:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.733644 10402 log.cc:824] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: Log is configured to *not* fsync() on all Append() calls
14:20:05.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.738107 10402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: No bootstrap required, opened a new log
14:20:05.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.762296 10402 raft_consensus.cc:348] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.762985 10402 raft_consensus.cc:374] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:05.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.763234 10402 raft_consensus.cc:729] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: be7085ccc0ce4009aa50e6ac7ae00c8b, State: Initialized, Role: FOLLOWER
14:20:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.763888 10402 consensus_queue.cc:260] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.766182 10411 sys_catalog.cc:455] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.766868 10411 sys_catalog.cc:458] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: This master's current role is: FOLLOWER
14:20:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.767474 10402 sys_catalog.cc:564] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: configured and running, proceeding with master startup.
14:20:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.775429 10411 raft_consensus.cc:482] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.776144 10411 raft_consensus.cc:504] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.779603 10411 leader_election.cc:288] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 425008e39c9541d182ddf94d18376ec0 (127.9.230.190:42571), 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409)
14:20:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:05.795506 10422 catalog_manager.cc:1558] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:05.796049 10422 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:05.819955 10346 tablet.cc:2369] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:05.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.792492 10315 master_runner.cc:186] Error getting master registration for 127.9.230.190:42571: OK, instance_id { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" instance_seqno: 1714573202232512 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:20:05.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.826756 10315 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:05.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.836984 10228 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 6
14:20:05.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.859084 10228 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: Bootstrap starting.
14:20:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.866130 10228 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: Neither blocks nor log segments found. Creating new log.
14:20:05.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.868484 10228 log.cc:824] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: Log is configured to *not* fsync() on all Append() calls
14:20:05.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.879513 10228 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: No bootstrap required, opened a new log
14:20:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.911163 10228 raft_consensus.cc:348] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.912063 10228 raft_consensus.cc:374] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.912364 10228 raft_consensus.cc:729] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 425008e39c9541d182ddf94d18376ec0, State: Initialized, Role: FOLLOWER
14:20:05.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.913475 10228 consensus_queue.cc:260] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.916805 10426 sys_catalog.cc:455] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:05.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.917645 10426 sys_catalog.cc:458] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: This master's current role is: FOLLOWER
14:20:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.918645 10228 sys_catalog.cc:564] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: configured and running, proceeding with master startup.
14:20:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.918905 10193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "425008e39c9541d182ddf94d18376ec0" is_pre_election: true
14:20:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.919826 10193 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate be7085ccc0ce4009aa50e6ac7ae00c8b in term 0.
14:20:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.921370 10342 leader_election.cc:302] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 425008e39c9541d182ddf94d18376ec0, be7085ccc0ce4009aa50e6ac7ae00c8b; no voters:
14:20:05.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.922613 10411 raft_consensus.cc:2781] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Leader pre-election won for term 1
14:20:05.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.922972 10411 raft_consensus.cc:482] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:05.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.923338 10411 raft_consensus.cc:3037] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 0 FOLLOWER]: Advancing to term 1
14:20:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:05.930537 10172 tablet.cc:2369] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:05.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.933562 10411 raft_consensus.cc:504] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.945266 10193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "425008e39c9541d182ddf94d18376ec0"
14:20:05.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.947245 10193 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 0 FOLLOWER]: Advancing to term 1
14:20:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.946007 10411 leader_election.cc:288] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [CANDIDATE]: Term 1 election: Requested vote from peers 425008e39c9541d182ddf94d18376ec0 (127.9.230.190:42571), 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409)
14:20:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.954977 10193 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate be7085ccc0ce4009aa50e6ac7ae00c8b in term 1.
14:20:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.956337 10342 leader_election.cc:302] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 425008e39c9541d182ddf94d18376ec0, be7085ccc0ce4009aa50e6ac7ae00c8b; no voters:
14:20:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.957404 10411 raft_consensus.cc:2781] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 1 FOLLOWER]: Leader election won for term 1
14:20:05.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.968534 10411 raft_consensus.cc:686] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [term 1 LEADER]: Becoming Leader. State: Replica: be7085ccc0ce4009aa50e6ac7ae00c8b, State: Running, Role: LEADER
14:20:05.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.969884 10411 consensus_queue.cc:237] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.983676 10433 sys_catalog.cc:455] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: SysCatalogTable state changed. Reason: New leader be7085ccc0ce4009aa50e6ac7ae00c8b. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.984793 10433 sys_catalog.cc:458] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: This master's current role is: LEADER
14:20:05.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.988799 10438 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
14:20:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:05.998131 10438 catalog_manager.cc:1484] Initializing Kudu cluster ID...
14:20:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.001879 10440 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:20:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.019562 10193 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Refusing update from remote peer be7085ccc0ce4009aa50e6ac7ae00c8b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:20:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.021591 10411 consensus_queue.cc:1035] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.011587 10150 master_runner.cc:186] Error getting master registration for 127.9.230.189:41409: OK, instance_id { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" instance_seqno: 1714573203927294 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:20:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.058373 10150 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:06.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.066936 10426 sys_catalog.cc:455] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader be7085ccc0ce4009aa50e6ac7ae00c8b. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.068097 10441 mvcc.cc:205] Tried to move back new op lower bound from 7022891851831685120 to 7022891851665227776. Current Snapshot: MvccSnapshot[applied={T|T < 7022891851831685120}]
14:20:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.069082 10426 sys_catalog.cc:458] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.074091 10433 sys_catalog.cc:455] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.074903 10433 sys_catalog.cc:458] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: This master's current role is: LEADER
14:20:06.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.076408 10442 mvcc.cc:205] Tried to move back new op lower bound from 7022891851831685120 to 7022891851665227776. Current Snapshot: MvccSnapshot[applied={T|T < 7022891851831685120}]
14:20:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.082150 10438 catalog_manager.cc:1347] Generated new cluster ID: 6dfa6e5a541d4958bd5614f0d7dbf206
14:20:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.082476 10438 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
14:20:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.082826 10445 sys_catalog.cc:455] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.083657 10445 sys_catalog.cc:458] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.088842 10344 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.9.230.189:41409 timed out after 0.310s (SENT)
14:20:06.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.119125 10438 catalog_manager.cc:1370] Generated new certificate authority record
14:20:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.121361 10438 catalog_manager.cc:1504] Loading token signing keys...
14:20:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.151014 10438 catalog_manager.cc:5857] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b: Generated new TSK 0
14:20:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.152107 10438 catalog_manager.cc:1514] Initializing in-progress tserver states...
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.175024 10138 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/logs
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data/info.pb
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.129:0
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.230.129
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.129
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:20:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:20:06.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.296636 10313 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } attempt: 7
14:20:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.370388 10313 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: Bootstrap starting.
14:20:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.379036 10313 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: Neither blocks nor log segments found. Creating new log.
14:20:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.381698 10313 log.cc:824] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: Log is configured to *not* fsync() on all Append() calls
14:20:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.401196 10313 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: No bootstrap required, opened a new log
14:20:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.404320 10344 leader_election.cc:334] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409): Timed out: RequestConsensusVote RPC to 127.9.230.189:41409 timed out after 0.310s (SENT)
14:20:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.438519 10313 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.440016 10313 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.440343 10313 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a1bad1ccc4b4beaa9e0dfdf3fef9185, State: Initialized, Role: FOLLOWER
14:20:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.441277 10313 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.448150 10313 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: configured and running, proceeding with master startup.
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.448329 10278 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" is_pre_election: true
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.449251 10278 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate be7085ccc0ce4009aa50e6ac7ae00c8b in term 0.
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.449796 10278 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34774 (request call id 1) took 670 ms (client timeout 310 ms). Trace:
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.450158 10278 rpcz_store.cc:269] 0501 14:20:05.779052 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:05.779226 (+ 174us) service_pool.cc:225] Handling call
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:06.449724 (+670498us) inbound_call.cc:173] Queueing success response
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.451246 10275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" is_pre_election: true
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.451906 10275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate be7085ccc0ce4009aa50e6ac7ae00c8b in term 0.
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.452410 10275 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34774 (request call id 4) took 358 ms (client timeout 310 ms). Trace:
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.452741 10275 rpcz_store.cc:269] 0501 14:20:06.094373 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:06.094511 (+ 138us) service_pool.cc:225] Handling call
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:06.452350 (+357839us) inbound_call.cc:173] Queueing success response
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.453943 10276 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 0 FOLLOWER]: Advancing to term 1
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.456931 10277 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.457948 10277 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34774 (request call id 2) took 511 ms (client timeout 325 ms). Trace:
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.458287 10277 rpcz_store.cc:269] 0501 14:20:05.946596 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:05.946780 (+ 184us) service_pool.cc:225] Handling call
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:06.457888 (+511108us) inbound_call.cc:173] Queueing success response
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.459074 10274 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.466923 10276 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Refusing update from remote peer be7085ccc0ce4009aa50e6ac7ae00c8b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:20:06.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.468978 10433 consensus_queue.cc:1035] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:06.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.447614 10451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.478902 10451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.505642 10458 mvcc.cc:205] Tried to move back new op lower bound from 7022891851831685120 to 7022891851665227776. Current Snapshot: MvccSnapshot[applied={T|T < 7022891851831685120}]
14:20:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.525432 10451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: SysCatalogTable state changed. Reason: New leader be7085ccc0ce4009aa50e6ac7ae00c8b. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.526230 10451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.527227 10455 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.528227 10455 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.560812 10467 catalog_manager.cc:1259] Loaded cluster ID: 6dfa6e5a541d4958bd5614f0d7dbf206
14:20:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.561075 10467 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: loading cluster ID for follower catalog manager: success
14:20:06.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.569329 10467 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: acquiring CA information for follower catalog manager: success
14:20:06.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.575424 10467 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:20:06.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.586951 10433 sys_catalog.cc:455] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:06.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.587991 10433 sys_catalog.cc:458] T 00000000000000000000000000000000 P be7085ccc0ce4009aa50e6ac7ae00c8b [sys.catalog]: This master's current role is: LEADER
14:20:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.628978 10448 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.629604 10448 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:06.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.630261 10448 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.681555 10448 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.682746 10448 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:20:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.683205 10448 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:20:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.684226 10448 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:20:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.685644 10448 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.230.129
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.748499 10448 tablet_server_runner.cc:78] Tablet server non-default flags:
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.129:0
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data/info.pb
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.129
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.230.129
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/logs
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 13:43:20 UTC on 1a15035a6830
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 76
14:20:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.751519 10448 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:06.753582 10448 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.774930 10475 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:07.031949 10440 catalog_manager.cc:1259] Loaded cluster ID: 6dfa6e5a541d4958bd5614f0d7dbf206
14:20:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:07.032467 10440 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: loading cluster ID for follower catalog manager: success
14:20:07.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:07.058619 10440 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: acquiring CA information for follower catalog manager: success
14:20:07.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:07.077332 10440 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:20:08.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:08.172158 10474 debug-util.cc:398] Leaking SignalData structure 0x7b0800014e80 after lost signal to thread 10448
14:20:08.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:08.816857 10448 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.044s user 0.366s sys 0.725s
14:20:08.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:06.776408 10476 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:08.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:08.817265 10448 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.045s user 0.366s sys 0.725s
14:20:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.826893 10448 server_base.cc:1034] running on GCE node
14:20:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:08.827540 10481 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:08.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.828886 10448 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.843298 10448 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.844810 10448 hybrid_clock.cc:648] HybridClock initialized: now 1714573208844761 us; error 59 us; skew 500 ppm
14:20:08.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.845976 10448 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:08.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.865677 10448 webserver.cc:460] Webserver started at http://127.9.230.129:38255/ using document root <none> and password file <none>
14:20:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.867009 10448 fs_manager.cc:359] Metadata directory not provided
14:20:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.867336 10448 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:08.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.867926 10448 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:20:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.875155 10448 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data/instance:
14:20:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e56d440a27b4c849965b6474097b000"
14:20:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:08 on dist-test-slave-kwk7"
14:20:08.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.876683 10448 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal/instance:
14:20:08.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e56d440a27b4c849965b6474097b000"
14:20:08.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:08 on dist-test-slave-kwk7"
14:20:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.891897 10448 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.013s sys 0.000s
14:20:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.905156 10487 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:08.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.906584 10448 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.011s sys 0.000s
14:20:08.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.906996 10448 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data,/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal
14:20:08.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e56d440a27b4c849965b6474097b000"
14:20:08.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:08 on dist-test-slave-kwk7"
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:08.907461 10448 fs_report.cc:383] FS layout report
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data/data
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.006075 10448 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.008039 10448 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.008620 10448 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.050333 10448 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:20:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.050647 10448 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:20:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.050961 10448 ts_tablet_manager.cc:590] Registered 0 tablets
14:20:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.051177 10448 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:20:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.514995 10448 rpc_server.cc:307] RPC server started. Bound to: 127.9.230.129:37055
14:20:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.515094 10611 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.230.129:37055 every 8 connection(s)
14:20:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.519004 10448 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/data/info.pb
14:20:09.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.528374 10138 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdRM744/build/tsan/bin/kudu as pid 10448
14:20:09.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.530134 10138 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-0/wal/instance
14:20:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.549430 10615 heartbeater.cc:347] Connected to a master server at 127.9.230.190:42571
14:20:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.558552 10615 heartbeater.cc:464] Registering TS with master...
14:20:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.560189 10615 heartbeater.cc:511] Master 127.9.230.190:42571 requested a full tablet report, sending...
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.563237 10138 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/logs
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data/info.pb
14:20:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.130:0
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.230.130
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.130
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:20:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:20:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:20:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.563894 10183 ts_manager.cc:194] Registered new tserver with Master: 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055)
14:20:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.557263 10612 heartbeater.cc:347] Connected to a master server at 127.9.230.188:32935
14:20:09.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.567809 10612 heartbeater.cc:464] Registering TS with master...
14:20:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.568917 10612 heartbeater.cc:511] Master 127.9.230.188:32935 requested a full tablet report, sending...
14:20:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.571964 10357 ts_manager.cc:194] Registered new tserver with Master: 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055)
14:20:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.574899 10357 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.230.129:34389
14:20:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.585914 10613 heartbeater.cc:347] Connected to a master server at 127.9.230.189:41409
14:20:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.586333 10613 heartbeater.cc:464] Registering TS with master...
14:20:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.587141 10613 heartbeater.cc:511] Master 127.9.230.189:41409 requested a full tablet report, sending...
14:20:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.592233 10268 ts_manager.cc:194] Registered new tserver with Master: 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055)
14:20:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.915649 10617 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.916193 10617 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.916918 10617 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.951898 10617 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.952610 10617 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:20:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.952925 10617 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:20:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.953157 10617 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:20:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:09.953692 10617 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.230.130
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.988559 10617 tablet_server_runner.cc:78] Tablet server non-default flags:
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.130:0
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data/info.pb
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.130
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.230.130
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/logs
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 13:43:20 UTC on 1a15035a6830
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 76
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.990159 10617 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:09.992112 10617 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:10.011497 10624 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:10.581538 10612 heartbeater.cc:503] Master 127.9.230.188:32935 was elected leader, sending a full tablet report...
14:20:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:10.011956 10623 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:10.012949 10617 server_base.cc:1034] running on GCE node
14:20:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:10.013330 10626 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.161262 10617 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:11.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.163486 10617 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.164844 10617 hybrid_clock.cc:648] HybridClock initialized: now 1714573211164799 us; error 49 us; skew 500 ppm
14:20:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.165657 10617 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:11.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.171648 10617 webserver.cc:460] Webserver started at http://127.9.230.130:35839/ using document root <none> and password file <none>
14:20:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.172516 10617 fs_manager.cc:359] Metadata directory not provided
14:20:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.172750 10617 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.173202 10617 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:20:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.178141 10617 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data/instance:
14:20:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3aa5299cc1ff47f1b7cd39baadf9db57"
14:20:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:11 on dist-test-slave-kwk7"
14:20:11.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.179348 10617 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal/instance:
14:20:11.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3aa5299cc1ff47f1b7cd39baadf9db57"
14:20:11.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:11 on dist-test-slave-kwk7"
14:20:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.187611 10617 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
14:20:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.193432 10637 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.194521 10617 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.194870 10617 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data,/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3aa5299cc1ff47f1b7cd39baadf9db57"
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:11 on dist-test-slave-kwk7"
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.195217 10617 fs_report.cc:383] FS layout report
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data/data
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.244141 10617 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:11.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.245733 10617 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:11.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.246232 10617 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.263378 10617 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:20:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.264045 10617 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
14:20:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.264564 10617 ts_tablet_manager.cc:590] Registered 0 tablets
14:20:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.264961 10617 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:20:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.459419 10617 rpc_server.cc:307] RPC server started. Bound to: 127.9.230.130:42049
14:20:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.459941 10761 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.230.130:42049 every 8 connection(s)
14:20:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.461906 10617 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/data/info.pb
14:20:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.471783 10138 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdRM744/build/tsan/bin/kudu as pid 10617
14:20:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.472631 10138 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-1/wal/instance
14:20:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.478960 10762 heartbeater.cc:347] Connected to a master server at 127.9.230.188:32935
14:20:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.483251 10762 heartbeater.cc:464] Registering TS with master...
14:20:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.484265 10764 heartbeater.cc:347] Connected to a master server at 127.9.230.190:42571
14:20:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.484660 10764 heartbeater.cc:464] Registering TS with master...
14:20:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.485916 10764 heartbeater.cc:511] Master 127.9.230.190:42571 requested a full tablet report, sending...
14:20:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.484679 10762 heartbeater.cc:511] Master 127.9.230.188:32935 requested a full tablet report, sending...
14:20:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.486117 10763 heartbeater.cc:347] Connected to a master server at 127.9.230.189:41409
14:20:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.486506 10763 heartbeater.cc:464] Registering TS with master...
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.486861 10138 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdRM744/build/tsan/bin/kudu
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/logs
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data/info.pb
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.131:0
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.230.131
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.131
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:20:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:20:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:20:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.487504 10763 heartbeater.cc:511] Master 127.9.230.189:41409 requested a full tablet report, sending...
14:20:11.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.489043 10357 ts_manager.cc:194] Registered new tserver with Master: 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049)
14:20:11.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.489944 10183 ts_manager.cc:194] Registered new tserver with Master: 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049)
14:20:11.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.490175 10268 ts_manager.cc:194] Registered new tserver with Master: 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049)
14:20:11.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.490958 10357 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.230.130:49581
14:20:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.800887 10767 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.801406 10767 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.802031 10767 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.833700 10767 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.834501 10767 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:20:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.834795 10767 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:20:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.835047 10767 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:20:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.835642 10767 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.230.131
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.871238 10767 tablet_server_runner.cc:78] Tablet server non-default flags:
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.230.148:43575
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.230.131:0
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data/info.pb
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.230.131
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:20:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.230.131
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/logs
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 13:43:20 UTC on 1a15035a6830
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 76
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.872365 10767 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:11.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.873912 10767 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.891340 10774 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:12.494904 10762 heartbeater.cc:503] Master 127.9.230.188:32935 was elected leader, sending a full tablet report...
14:20:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.891433 10776 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:11.892092 10773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:11.892488 10767 server_base.cc:1034] running on GCE node
14:20:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.022794 10767 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:13.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.025391 10767 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.026835 10767 hybrid_clock.cc:648] HybridClock initialized: now 1714573213026779 us; error 61 us; skew 500 ppm
14:20:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.027737 10767 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.038770 10767 webserver.cc:460] Webserver started at http://127.9.230.131:45387/ using document root <none> and password file <none>
14:20:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.039788 10767 fs_manager.cc:359] Metadata directory not provided
14:20:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.040024 10767 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.040514 10767 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:20:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.045871 10767 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data/instance:
14:20:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f785f90643644d76b1f59b3b3907141d"
14:20:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:13 on dist-test-slave-kwk7"
14:20:13.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.047114 10767 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal/instance:
14:20:13.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f785f90643644d76b1f59b3b3907141d"
14:20:13.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:13 on dist-test-slave-kwk7"
14:20:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.054673 10767 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
14:20:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.060315 10784 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.061435 10767 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.061774 10767 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data,/tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f785f90643644d76b1f59b3b3907141d"
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 14:20:13 on dist-test-slave-kwk7"
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.062129 10767 fs_report.cc:383] FS layout report
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data/data
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.114245 10767 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.115731 10767 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.116241 10767 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.126925 10767 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:20:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.127161 10767 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:20:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.127483 10767 ts_tablet_manager.cc:590] Registered 0 tablets
14:20:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.127666 10767 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:20:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.307188 10767 rpc_server.cc:307] RPC server started. Bound to: 127.9.230.131:35983
14:20:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.307657 10908 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.230.131:35983 every 8 connection(s)
14:20:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.310801 10767 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/data/info.pb
14:20:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.316154 10138 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdRM744/build/tsan/bin/kudu as pid 10767
14:20:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.316603 10138 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdRM744/test-tmp/mini-kudu-cluster7391316455477666038/ts-2/wal/instance
14:20:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.324741 10911 heartbeater.cc:347] Connected to a master server at 127.9.230.190:42571
14:20:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.325575 10911 heartbeater.cc:464] Registering TS with master...
14:20:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.330561 10911 heartbeater.cc:511] Master 127.9.230.190:42571 requested a full tablet report, sending...
14:20:13.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.334216 10183 ts_manager.cc:194] Registered new tserver with Master: f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:13.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.336086 10909 heartbeater.cc:347] Connected to a master server at 127.9.230.188:32935
14:20:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.336612 10909 heartbeater.cc:464] Registering TS with master...
14:20:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.337782 10909 heartbeater.cc:511] Master 127.9.230.188:32935 requested a full tablet report, sending...
14:20:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.340749 10357 ts_manager.cc:194] Registered new tserver with Master: f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.342242 10357 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.230.131:35363
14:20:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.345374 10910 heartbeater.cc:347] Connected to a master server at 127.9.230.189:41409
14:20:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.345783 10910 heartbeater.cc:464] Registering TS with master...
14:20:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.346319 10910 heartbeater.cc:511] Master 127.9.230.189:41409 requested a full tablet report, sending...
14:20:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.349788 10268 ts_manager.cc:194] Registered new tserver with Master: f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.358227 10138 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
14:20:13.369 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:20:13.369 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:20:13.370 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.230.190"
port: 42571
}
bound_http_address {
host: "127.9.230.190"
port: 45901
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.230.189"
port: 41409
}
bound_http_address {
host: "127.9.230.189"
port: 38337
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.230.188"
port: 32935
}
bound_http_address {
host: "127.9.230.188"
port: 37275
}
}
}
14:20:13.370 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:20:13.371 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.9.230.129"
port: 37055
}
bound_http_address {
host: "127.9.230.129"
port: 38255
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.9.230.130"
port: 42049
}
bound_http_address {
host: "127.9.230.130"
port: 35839
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.9.230.131"
port: 35983
}
bound_http_address {
host: "127.9.230.131"
port: 45387
}
}
}
14:20:13.371 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:20:13.373 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:20:13.375 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.230.190:42571(127.9.230.190:42571)] connecting to peer
14:20:13.376 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.230.189:41409(127.9.230.189:41409)] connecting to peer
14:20:13.376 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:13.378 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.230.189:41409(127.9.230.189:41409)] Successfully connected to peer
14:20:13.380 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.230.190:42571(127.9.230.190:42571)] Successfully connected to peer
14:20:13.380 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.230.189
14:20:13.383 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.230.190
14:20:13.386 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.230.188:32935(127.9.230.188:32935)] Successfully connected to peer
14:20:13.394 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x81367093, L:/127.0.0.1:40228 - R:/127.9.230.189:41409] using SASL/PLAIN
14:20:13.396 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.230.188
14:20:13.399 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd210d24a, L:/127.0.0.1:45550 - R:/127.9.230.190:42571] using SASL/PLAIN
14:20:13.408 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb84b82bc, L:/127.0.0.1:55406 - R:/127.9.230.188:32935] using SASL/PLAIN
14:20:13.411 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:20:13.411 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=37), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [32ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [36ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: OK, deferred=null) after lookup
14:20:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.413720 10357 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55406:
14:20:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
14:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
14:20:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:13.418069 10357 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:20:13.465 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 503d3d4b9e2148e0b48028bb1ad7f904
14:20:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.538501 10547 tablet_service.cc:1467] Processing CreateTablet for tablet ded3a30847e24b8cae14ffd47ff6f377 (DEFAULT_TABLE table=txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:20:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.538555 10546 tablet_service.cc:1467] Processing CreateTablet for tablet 36fe95aae5bd403e8dba8bce7d5e00ea (DEFAULT_TABLE table=txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:20:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.540710 10547 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ded3a30847e24b8cae14ffd47ff6f377. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.541409 10546 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36fe95aae5bd403e8dba8bce7d5e00ea. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.548848 10843 tablet_service.cc:1467] Processing CreateTablet for tablet 36fe95aae5bd403e8dba8bce7d5e00ea (DEFAULT_TABLE table=txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:20:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.549039 10697 tablet_service.cc:1467] Processing CreateTablet for tablet ded3a30847e24b8cae14ffd47ff6f377 (DEFAULT_TABLE table=txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:20:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.550318 10696 tablet_service.cc:1467] Processing CreateTablet for tablet 36fe95aae5bd403e8dba8bce7d5e00ea (DEFAULT_TABLE table=txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:20:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.551339 10697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ded3a30847e24b8cae14ffd47ff6f377. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.552120 10696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36fe95aae5bd403e8dba8bce7d5e00ea. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.551752 10844 tablet_service.cc:1467] Processing CreateTablet for tablet ded3a30847e24b8cae14ffd47ff6f377 (DEFAULT_TABLE table=txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:20:13.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.550953 10843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36fe95aae5bd403e8dba8bce7d5e00ea. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:13.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.558954 10844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ded3a30847e24b8cae14ffd47ff6f377. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.581589 10935 tablet_bootstrap.cc:492] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000: Bootstrap starting.
14:20:13.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.588418 10935 tablet_bootstrap.cc:654] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000: Neither blocks nor log segments found. Creating new log.
14:20:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.592063 10936 tablet_bootstrap.cc:492] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: Bootstrap starting.
14:20:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.592998 10935 log.cc:824] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000: Log is configured to *not* fsync() on all Append() calls
14:20:13.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.599792 10936 tablet_bootstrap.cc:654] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: Neither blocks nor log segments found. Creating new log.
14:20:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.602835 10936 log.cc:824] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: Log is configured to *not* fsync() on all Append() calls
14:20:13.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.606155 10938 tablet_bootstrap.cc:492] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d: Bootstrap starting.
14:20:13.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.611300 10935 tablet_bootstrap.cc:492] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000: No bootstrap required, opened a new log
14:20:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.612010 10935 ts_tablet_manager.cc:1367] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000: Time spent bootstrapping tablet: real 0.031s user 0.023s sys 0.004s
14:20:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.612248 10936 tablet_bootstrap.cc:492] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: No bootstrap required, opened a new log
14:20:13.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.612782 10936 ts_tablet_manager.cc:1367] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: Time spent bootstrapping tablet: real 0.021s user 0.006s sys 0.009s
14:20:13.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.613097 10938 tablet_bootstrap.cc:654] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d: Neither blocks nor log segments found. Creating new log.
14:20:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.615401 10938 log.cc:824] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d: Log is configured to *not* fsync() on all Append() calls
14:20:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.620362 10938 tablet_bootstrap.cc:492] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d: No bootstrap required, opened a new log
14:20:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.620865 10938 ts_tablet_manager.cc:1367] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d: Time spent bootstrapping tablet: real 0.015s user 0.008s sys 0.007s
14:20:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.643901 10938 raft_consensus.cc:348] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.644213 10936 raft_consensus.cc:348] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.644953 10938 raft_consensus.cc:374] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.645228 10936 raft_consensus.cc:374] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.645254 10938 raft_consensus.cc:729] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f785f90643644d76b1f59b3b3907141d, State: Initialized, Role: FOLLOWER
14:20:13.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.645588 10936 raft_consensus.cc:729] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa5299cc1ff47f1b7cd39baadf9db57, State: Initialized, Role: FOLLOWER
14:20:13.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.646193 10938 consensus_queue.cc:260] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.646548 10936 consensus_queue.cc:260] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.649711 10935 raft_consensus.cc:348] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.650682 10935 raft_consensus.cc:374] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.650974 10935 raft_consensus.cc:729] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e56d440a27b4c849965b6474097b000, State: Initialized, Role: FOLLOWER
14:20:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.652051 10936 ts_tablet_manager.cc:1398] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: Time spent starting tablet: real 0.039s user 0.030s sys 0.007s
14:20:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.651859 10935 consensus_queue.cc:260] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.653751 10936 tablet_bootstrap.cc:492] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57: Bootstrap starting.
14:20:13.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.658128 10942 raft_consensus.cc:482] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:13.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.658391 10938 ts_tablet_manager.cc:1398] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d: Time spent starting tablet: real 0.037s user 0.019s sys 0.013s
14:20:13.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.659428 10909 heartbeater.cc:503] Master 127.9.230.188:32935 was elected leader, sending a full tablet report...
14:20:13.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.658977 10942 raft_consensus.cc:504] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.670746 10938 tablet_bootstrap.cc:492] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d: Bootstrap starting.
14:20:13.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.686278 10942 leader_election.cc:288] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055), f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:13.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.689184 10935 ts_tablet_manager.cc:1398] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000: Time spent starting tablet: real 0.077s user 0.036s sys 0.008s
14:20:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.690145 10935 tablet_bootstrap.cc:492] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000: Bootstrap starting.
14:20:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.690328 10936 tablet_bootstrap.cc:654] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57: Neither blocks nor log segments found. Creating new log.
14:20:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.695852 10938 tablet_bootstrap.cc:654] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d: Neither blocks nor log segments found. Creating new log.
14:20:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.696833 10935 tablet_bootstrap.cc:654] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000: Neither blocks nor log segments found. Creating new log.
14:20:13.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.716507 10864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36fe95aae5bd403e8dba8bce7d5e00ea" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f785f90643644d76b1f59b3b3907141d" is_pre_election: true
14:20:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:13.721832 10639 leader_election.cc:341] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983): Illegal state: must be running to vote when last-logged opid is not known
14:20:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:13.731086 10766 tablet.cc:2369] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:13.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.733839 10938 tablet_bootstrap.cc:492] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d: No bootstrap required, opened a new log
14:20:13.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.734248 10935 tablet_bootstrap.cc:492] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000: No bootstrap required, opened a new log
14:20:13.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.734696 10936 tablet_bootstrap.cc:492] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57: No bootstrap required, opened a new log
14:20:13.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.734730 10938 ts_tablet_manager.cc:1367] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d: Time spent bootstrapping tablet: real 0.064s user 0.017s sys 0.010s
14:20:13.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.735148 10936 ts_tablet_manager.cc:1367] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57: Time spent bootstrapping tablet: real 0.082s user 0.018s sys 0.007s
14:20:13.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.734653 10935 ts_tablet_manager.cc:1367] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000: Time spent bootstrapping tablet: real 0.045s user 0.015s sys 0.024s
14:20:13.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.737695 10936 raft_consensus.cc:348] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.738180 10938 raft_consensus.cc:348] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.739151 10938 raft_consensus.cc:374] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.739672 10938 raft_consensus.cc:729] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f785f90643644d76b1f59b3b3907141d, State: Initialized, Role: FOLLOWER
14:20:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.739370 10935 raft_consensus.cc:348] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.740039 10935 raft_consensus.cc:374] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.738431 10936 raft_consensus.cc:374] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.740298 10935 raft_consensus.cc:729] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e56d440a27b4c849965b6474097b000, State: Initialized, Role: FOLLOWER
14:20:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.740478 10936 raft_consensus.cc:729] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa5299cc1ff47f1b7cd39baadf9db57, State: Initialized, Role: FOLLOWER
14:20:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.740914 10935 consensus_queue.cc:260] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.741209 10567 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36fe95aae5bd403e8dba8bce7d5e00ea" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e56d440a27b4c849965b6474097b000" is_pre_election: true
14:20:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.741369 10936 consensus_queue.cc:260] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.741308 10938 consensus_queue.cc:260] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.742101 10567 raft_consensus.cc:2454] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 0.
14:20:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.743067 10935 ts_tablet_manager.cc:1398] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000: Time spent starting tablet: real 0.006s user 0.000s sys 0.004s
14:20:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.743497 10936 ts_tablet_manager.cc:1398] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57: Time spent starting tablet: real 0.008s user 0.006s sys 0.001s
14:20:13.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.749563 10938 ts_tablet_manager.cc:1398] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d: Time spent starting tablet: real 0.014s user 0.005s sys 0.000s
14:20:13.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.749866 10640 leader_election.cc:302] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3aa5299cc1ff47f1b7cd39baadf9db57, 8e56d440a27b4c849965b6474097b000; no voters: f785f90643644d76b1f59b3b3907141d
14:20:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.754256 10942 raft_consensus.cc:2781] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:20:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.754629 10942 raft_consensus.cc:482] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.754959 10942 raft_consensus.cc:3037] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Advancing to term 1
14:20:13.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.761371 10942 raft_consensus.cc:504] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.763195 10942 leader_election.cc:288] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 election: Requested vote from peers 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055), f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:13.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.764284 10864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36fe95aae5bd403e8dba8bce7d5e00ea" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f785f90643644d76b1f59b3b3907141d"
14:20:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.764566 10567 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36fe95aae5bd403e8dba8bce7d5e00ea" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e56d440a27b4c849965b6474097b000"
14:20:13.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.764922 10864 raft_consensus.cc:3037] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Advancing to term 1
14:20:13.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.765178 10567 raft_consensus.cc:3037] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Advancing to term 1
14:20:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.770810 10864 raft_consensus.cc:2454] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 1.
14:20:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.771534 10567 raft_consensus.cc:2454] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 1.
14:20:13.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.771888 10639 leader_election.cc:302] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa5299cc1ff47f1b7cd39baadf9db57, f785f90643644d76b1f59b3b3907141d; no voters:
14:20:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.773123 10942 raft_consensus.cc:2781] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 FOLLOWER]: Leader election won for term 1
14:20:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.776279 10942 raft_consensus.cc:686] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 LEADER]: Becoming Leader. State: Replica: 3aa5299cc1ff47f1b7cd39baadf9db57, State: Running, Role: LEADER
14:20:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.777330 10942 consensus_queue.cc:237] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:13.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.787397 10356 catalog_manager.cc:5484] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130). New cstate: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } health_report { overall_health: UNKNOWN } } }
14:20:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:13.802084 10616 tablet.cc:2369] T 36fe95aae5bd403e8dba8bce7d5e00ea P 8e56d440a27b4c849965b6474097b000: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:13.841940 10912 tablet.cc:2369] T 36fe95aae5bd403e8dba8bce7d5e00ea P f785f90643644d76b1f59b3b3907141d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.868214 10941 raft_consensus.cc:482] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.868711 10941 raft_consensus.cc:504] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.870990 10941 leader_election.cc:288] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055), 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049)
14:20:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.881147 10567 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ded3a30847e24b8cae14ffd47ff6f377" candidate_uuid: "f785f90643644d76b1f59b3b3907141d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e56d440a27b4c849965b6474097b000" is_pre_election: true
14:20:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.881218 10717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ded3a30847e24b8cae14ffd47ff6f377" candidate_uuid: "f785f90643644d76b1f59b3b3907141d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" is_pre_election: true
14:20:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.881672 10567 raft_consensus.cc:2454] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f785f90643644d76b1f59b3b3907141d in term 0.
14:20:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.881911 10717 raft_consensus.cc:2454] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f785f90643644d76b1f59b3b3907141d in term 0.
14:20:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.882828 10787 leader_election.cc:302] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8e56d440a27b4c849965b6474097b000, f785f90643644d76b1f59b3b3907141d; no voters:
14:20:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.883564 10941 raft_consensus.cc:2781] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:20:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.883827 10941 raft_consensus.cc:482] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.884100 10941 raft_consensus.cc:3037] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Advancing to term 1
14:20:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.888104 10941 raft_consensus.cc:504] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.889607 10941 leader_election.cc:288] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [CANDIDATE]: Term 1 election: Requested vote from peers 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055), 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049)
14:20:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.890482 10567 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ded3a30847e24b8cae14ffd47ff6f377" candidate_uuid: "f785f90643644d76b1f59b3b3907141d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e56d440a27b4c849965b6474097b000"
14:20:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.890674 10717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ded3a30847e24b8cae14ffd47ff6f377" candidate_uuid: "f785f90643644d76b1f59b3b3907141d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57"
14:20:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.890964 10567 raft_consensus.cc:3037] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Advancing to term 1
14:20:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.891067 10717 raft_consensus.cc:3037] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Advancing to term 1
14:20:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.895087 10567 raft_consensus.cc:2454] T ded3a30847e24b8cae14ffd47ff6f377 P 8e56d440a27b4c849965b6474097b000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f785f90643644d76b1f59b3b3907141d in term 1.
14:20:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.895579 10717 raft_consensus.cc:2454] T ded3a30847e24b8cae14ffd47ff6f377 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f785f90643644d76b1f59b3b3907141d in term 1.
14:20:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.896178 10787 leader_election.cc:302] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8e56d440a27b4c849965b6474097b000, f785f90643644d76b1f59b3b3907141d; no voters:
14:20:13.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.896831 10941 raft_consensus.cc:2781] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 1 FOLLOWER]: Leader election won for term 1
14:20:13.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.898134 10941 raft_consensus.cc:686] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [term 1 LEADER]: Becoming Leader. State: Replica: f785f90643644d76b1f59b3b3907141d, State: Running, Role: LEADER
14:20:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.899708 10941 consensus_queue.cc:237] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } }
14:20:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:13.908383 10356 catalog_manager.cc:5484] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d reported cstate change: term changed from 0 to 1, leader changed from <none> to f785f90643644d76b1f59b3b3907141d (127.9.230.131). New cstate: current_term: 1 leader_uuid: "f785f90643644d76b1f59b3b3907141d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } health_report { overall_health: UNKNOWN } } }
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.005553 10356 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55420:
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:20:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.007992 10356 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:20:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.038851 10546 tablet_service.cc:1467] Processing CreateTablet for tablet e971f7d692c542f981fc9f70af9ab1d8 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=011b239f4d6c4a64b867f3f9fada3152]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:20:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.040249 10546 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e971f7d692c542f981fc9f70af9ab1d8. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.039875 10697 tablet_service.cc:1467] Processing CreateTablet for tablet e971f7d692c542f981fc9f70af9ab1d8 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=011b239f4d6c4a64b867f3f9fada3152]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:20:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.041523 10697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e971f7d692c542f981fc9f70af9ab1d8. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.041554 10843 tablet_service.cc:1467] Processing CreateTablet for tablet e971f7d692c542f981fc9f70af9ab1d8 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=011b239f4d6c4a64b867f3f9fada3152]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:20:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.043133 10843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e971f7d692c542f981fc9f70af9ab1d8. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.057020 10936 tablet_bootstrap.cc:492] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: Bootstrap starting.
14:20:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.057129 10935 tablet_bootstrap.cc:492] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: Bootstrap starting.
14:20:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.060746 10938 tablet_bootstrap.cc:492] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: Bootstrap starting.
14:20:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.063048 10936 tablet_bootstrap.cc:654] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: Neither blocks nor log segments found. Creating new log.
14:20:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.063058 10935 tablet_bootstrap.cc:654] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: Neither blocks nor log segments found. Creating new log.
14:20:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.066031 10938 tablet_bootstrap.cc:654] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: Neither blocks nor log segments found. Creating new log.
14:20:14.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.072557 10936 tablet_bootstrap.cc:492] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: No bootstrap required, opened a new log
14:20:14.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.073295 10936 ts_tablet_manager.cc:1367] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.007s
14:20:14.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.075302 10935 tablet_bootstrap.cc:492] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: No bootstrap required, opened a new log
14:20:14.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.075626 10938 tablet_bootstrap.cc:492] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: No bootstrap required, opened a new log
14:20:14.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.075729 10935 ts_tablet_manager.cc:1367] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: Time spent bootstrapping tablet: real 0.019s user 0.008s sys 0.008s
14:20:14.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.076018 10938 ts_tablet_manager.cc:1367] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.008s
14:20:14.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.075901 10936 raft_consensus.cc:348] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.076516 10936 raft_consensus.cc:374] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:14.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.076867 10936 raft_consensus.cc:729] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa5299cc1ff47f1b7cd39baadf9db57, State: Initialized, Role: FOLLOWER
14:20:14.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.077489 10936 consensus_queue.cc:260] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.078358 10935 raft_consensus.cc:348] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.078516 10938 raft_consensus.cc:348] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.078604 10942 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.079144 10935 raft_consensus.cc:374] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.079171 10938 raft_consensus.cc:374] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.079396 10942 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.079504 10936 ts_tablet_manager.cc:1398] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.079437 10935 raft_consensus.cc:729] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e56d440a27b4c849965b6474097b000, State: Initialized, Role: FOLLOWER
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.079495 10938 raft_consensus.cc:729] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f785f90643644d76b1f59b3b3907141d, State: Initialized, Role: FOLLOWER
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.080134 10935 consensus_queue.cc:260] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.080085 10938 consensus_queue.cc:260] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.081591 10943 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.082304 10935 ts_tablet_manager.cc:1398] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: Time spent starting tablet: real 0.006s user 0.007s sys 0.000s
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.082460 10943 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.084673 10938 ts_tablet_manager.cc:1398] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: Time spent starting tablet: real 0.008s user 0.005s sys 0.000s
14:20:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.087924 10942 raft_consensus.cc:482] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.088342 10942 raft_consensus.cc:504] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.088903 10941 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.089702 10941 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.091708 10942 leader_election.cc:288] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055), f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:14.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.091892 10567 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e971f7d692c542f981fc9f70af9ab1d8" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e56d440a27b4c849965b6474097b000" is_pre_election: true
14:20:14.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.092509 10567 raft_consensus.cc:2454] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 0.
14:20:14.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.092912 10864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e971f7d692c542f981fc9f70af9ab1d8" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f785f90643644d76b1f59b3b3907141d" is_pre_election: true
14:20:14.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.093544 10864 raft_consensus.cc:2454] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 0.
14:20:14.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.094036 10640 leader_election.cc:302] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa5299cc1ff47f1b7cd39baadf9db57, 8e56d440a27b4c849965b6474097b000; no voters:
14:20:14.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.095187 10942 raft_consensus.cc:2781] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:20:14.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.096956 10942 raft_consensus.cc:482] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:14.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.097291 10942 raft_consensus.cc:3037] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 0 FOLLOWER]: Advancing to term 1
14:20:14.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.104177 10942 raft_consensus.cc:504] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.106715 10567 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e971f7d692c542f981fc9f70af9ab1d8" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e56d440a27b4c849965b6474097b000"
14:20:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.106891 10864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e971f7d692c542f981fc9f70af9ab1d8" candidate_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f785f90643644d76b1f59b3b3907141d"
14:20:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.107196 10942 leader_election.cc:288] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 election: Requested vote from peers 8e56d440a27b4c849965b6474097b000 (127.9.230.129:37055), f785f90643644d76b1f59b3b3907141d (127.9.230.131:35983)
14:20:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.107296 10567 raft_consensus.cc:3037] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 0 FOLLOWER]: Advancing to term 1
14:20:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.107379 10864 raft_consensus.cc:3037] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 0 FOLLOWER]: Advancing to term 1
14:20:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.112066 10567 raft_consensus.cc:2454] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 1.
14:20:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.112303 10864 raft_consensus.cc:2454] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa5299cc1ff47f1b7cd39baadf9db57 in term 1.
14:20:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.112967 10640 leader_election.cc:302] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa5299cc1ff47f1b7cd39baadf9db57, 8e56d440a27b4c849965b6474097b000; no voters:
14:20:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.113595 10942 raft_consensus.cc:2781] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 FOLLOWER]: Leader election won for term 1
14:20:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.113901 10942 raft_consensus.cc:686] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [term 1 LEADER]: Becoming Leader. State: Replica: 3aa5299cc1ff47f1b7cd39baadf9db57, State: Running, Role: LEADER
14:20:14.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.114583 10942 consensus_queue.cc:237] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } }
14:20:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.119623 10949 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: TxnStatusTablet state changed. Reason: New leader 3aa5299cc1ff47f1b7cd39baadf9db57. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.120455 10949 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: This TxnStatusTablet replica's current role is: LEADER
14:20:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.124547 10356 catalog_manager.cc:5484] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130). New cstate: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } health_report { overall_health: UNKNOWN } } }
14:20:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.125957 10971 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
14:20:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.126483 10971 txn_status_manager.cc:929] Loading transaction status metadata into memory...
14:20:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.182552 10864 raft_consensus.cc:1261] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d [term 1 FOLLOWER]: Refusing update from remote peer 3aa5299cc1ff47f1b7cd39baadf9db57: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:20:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.183081 10567 raft_consensus.cc:1261] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000 [term 1 FOLLOWER]: Refusing update from remote peer 3aa5299cc1ff47f1b7cd39baadf9db57: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:20:14.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.184042 10942 consensus_queue.cc:1035] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:14.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.184772 10949 consensus_queue.cc:1035] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.204658 10941 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: TxnStatusTablet state changed. Reason: New leader 3aa5299cc1ff47f1b7cd39baadf9db57. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.205430 10941 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.206545 10942 consensus_queue.cc:1035] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.222421 10975 mvcc.cc:205] Tried to move back new op lower bound from 7022891885276917760 to 7022891885019582464. Current Snapshot: MvccSnapshot[applied={T|T < 7022891885276917760}]
14:20:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.229552 10942 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.230367 10942 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: This TxnStatusTablet replica's current role is: LEADER
14:20:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.238543 10943 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: TxnStatusTablet state changed. Reason: New leader 3aa5299cc1ff47f1b7cd39baadf9db57. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.239598 10943 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.256 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
14:20:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.280251 10949 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.282749 10949 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 3aa5299cc1ff47f1b7cd39baadf9db57: This TxnStatusTablet replica's current role is: LEADER
14:20:14.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.285404 10941 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.286125 10941 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P f785f90643644d76b1f59b3b3907141d: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.295 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 503d3d4b9e2148e0b48028bb1ad7f904
14:20:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.302292 10987 mvcc.cc:205] Tried to move back new op lower bound from 7022891885276917760 to 7022891885019582464. Current Snapshot: MvccSnapshot[applied={T|T < 7022891885276917760}]
14:20:14.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.319579 10942 consensus_queue.cc:1035] T 36fe95aae5bd403e8dba8bce7d5e00ea P 3aa5299cc1ff47f1b7cd39baadf9db57 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:14.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet ded3a30847e24b8cae14ffd47ff6f377 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
14:20:14.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 36fe95aae5bd403e8dba8bce7d5e00ea for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
14:20:14.324 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=ded3a30847e24b8cae14ffd47ff6f377@[3aa5299cc1ff47f1b7cd39baadf9db57(127.9.230.130:42049),8e56d440a27b4c849965b6474097b000(127.9.230.129:37055),f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=36fe95aae5bd403e8dba8bce7d5e00ea@[3aa5299cc1ff47f1b7cd39baadf9db57(127.9.230.130:42049)[L],8e56d440a27b4c849965b6474097b000(127.9.230.129:37055),f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
14:20:14.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.230.188:32935, [28ms] Sub RPC GetTableLocations: received response from server master-127.9.230.188:32935: OK, deferred=null) after lookup
14:20:14.325 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)] connecting to peer
14:20:14.326 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)] Successfully connected to peer
14:20:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.382035 10943 tablet_replica.cc:439] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 } } peers { permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 } } peers { permanent_uuid: "f785f90643644d76b1f59b3b3907141d" member_type: VOTER last_known_addr { host: "127.9.230.131" port: 35983 } } }
14:20:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.382750 10943 tablet_replica.cc:442] T e971f7d692c542f981fc9f70af9ab1d8 P 8e56d440a27b4c849965b6474097b000: This TxnStatusTablet replica's current role is: FOLLOWER
14:20:14.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.385818 10941 consensus_queue.cc:1035] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3aa5299cc1ff47f1b7cd39baadf9db57" member_type: VOTER last_known_addr { host: "127.9.230.130" port: 42049 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.505300 10991 consensus_queue.cc:1035] T ded3a30847e24b8cae14ffd47ff6f377 P f785f90643644d76b1f59b3b3907141d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e56d440a27b4c849965b6474097b000" member_type: VOTER last_known_addr { host: "127.9.230.129" port: 37055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:20:14.556 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x05b45a61, L:/127.0.0.1:46140 - R:/127.9.230.131:35983] using TOKEN/null
14:20:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.559056 10785 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.9.230.130:42049, user_credentials={real_user=slave}} blocked reactor thread for 96878.2us
14:20:14.685 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.230.188:32935
14:20:14.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:20:14.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.686328 10138 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdRM744/build/tsan/bin/kudu with pid 10315
14:20:14.714 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.230.188:32935(127.9.230.188:32935)] handling channelInactive
14:20:14.714 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state READY due to: connection closed
14:20:14.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:14.775 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.776 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.777 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.9.230.188:32935(127.9.230.188:32935) for tablet Kudu Master: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.795 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.796 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.796 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.797 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.799 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.815 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.816 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.816 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.817 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.818 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.835 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.838 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.855 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.856 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.856 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.858 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.895 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.896 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.897 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.898 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.915 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.920 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.919831 10997 raft_consensus.cc:482] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader be7085ccc0ce4009aa50e6ac7ae00c8b)
14:20:14.920 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=145), Traces: [0ms] sending RPC to server master-127.9.230.188:32935, [2ms] received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [21ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [41ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [61ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [121ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [122ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [144ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@1469980348(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1799166939@321c565b -> (continuation of Deferred@1291058157 after retry RPC after error@330533465) -> (continuation of Deferred@880745837 after retry RPC after error@1686186807) -> (continuation of Deferred@931039449 after retry RPC after error@940584237) -> (continuation of Deferred@1959320375 after retry RPC after error@436047370) -> (continuation of Deferred@1510405548 after retry RPC after error@1665585796), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/792392946@51e5bcb8 -> (continuation of Deferred@1291058157 after retry RPC after error@330533465) -> (continuation of Deferred@880745837 after retry RPC after error@1686186807) -> (continuation of Deferred@931039449 after retry RPC after error@940584237) -> (continuation of Deferred@1959320375 after retry RPC after error@436047370) -> (continuation of Deferred@1510405548 after retry RPC after error@1665585796)))
14:20:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.920675 10997 raft_consensus.cc:504] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:14.921 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=145), Traces: [0ms] sending RPC to server master-127.9.230.188:32935, [2ms] received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [21ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [41ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [61ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [121ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [122ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [144ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@1469980348(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1799166939@321c565b -> (continuation of Deferred@1291058157 after retry RPC after error@330533465) -> (continuation of Deferred@880745837 after retry RPC after error@1686186807) -> (continuation of Deferred@931039449 after retry RPC after error@940584237) -> (continuation of Deferred@1959320375 after retry RPC after error@436047370) -> (continuation of Deferred@1510405548 after retry RPC after error@1665585796), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/792392946@51e5bcb8 -> (continuation of Deferred@1291058157 after retry RPC after error@330533465) -> (continuation of Deferred@880745837 after retry RPC after error@1686186807) -> (continuation of Deferred@931039449 after retry RPC after error@940584237) -> (continuation of Deferred@1959320375 after retry RPC after error@436047370) -> (continuation of Deferred@1510405548 after retry RPC after error@1665585796)))
14:20:14.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.923324 10998 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader be7085ccc0ce4009aa50e6ac7ae00c8b)
14:20:14.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.923954 10998 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.924608 10274 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "425008e39c9541d182ddf94d18376ec0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" is_pre_election: true
14:20:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.925654 10274 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 425008e39c9541d182ddf94d18376ec0 in term 1.
14:20:14.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.927461 10998 leader_election.cc:288] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 425008e39c9541d182ddf94d18376ec0 (127.9.230.190:42571), be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935)
14:20:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.927412 10193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "425008e39c9541d182ddf94d18376ec0" is_pre_election: true
14:20:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.928102 10193 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a1bad1ccc4b4beaa9e0dfdf3fef9185 in term 1.
14:20:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.928450 10170 leader_election.cc:302] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a1bad1ccc4b4beaa9e0dfdf3fef9185, 425008e39c9541d182ddf94d18376ec0; no voters:
14:20:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.931787 10997 leader_election.cc:288] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409), be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935)
14:20:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.933135 10997 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:20:14.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.934134 10997 raft_consensus.cc:482] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Starting leader election (detected failure of leader be7085ccc0ce4009aa50e6ac7ae00c8b)
14:20:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.938563 10997 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 1 FOLLOWER]: Advancing to term 2
14:20:14.950 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.950803 10997 raft_consensus.cc:504] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:14.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.953279 10997 leader_election.cc:288] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 2 election: Requested vote from peers 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409), be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935)
14:20:14.961 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.962 [INFO - pool-46-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.962 [WARN - pool-46-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.962980 10762 heartbeater.cc:644] Failed to heartbeat to 127.9.230.188:32935 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.965327 10253 leader_election.cc:302] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a1bad1ccc4b4beaa9e0dfdf3fef9185, 425008e39c9541d182ddf94d18376ec0; no voters:
14:20:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.965857 10252 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111) [suppressed 6 similar messages]
14:20:14.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.966236 10998 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:20:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.966630 10998 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Starting leader election (detected failure of leader be7085ccc0ce4009aa50e6ac7ae00c8b)
14:20:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.966955 10998 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 1 FOLLOWER]: Advancing to term 2
14:20:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.967579 10167 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111) [suppressed 12 similar messages]
14:20:14.975 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.975342 10998 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:14.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.982533 10998 leader_election.cc:288] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [CANDIDATE]: Term 2 election: Requested vote from peers 425008e39c9541d182ddf94d18376ec0 (127.9.230.190:42571), be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935)
14:20:14.983 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.984194 10193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "425008e39c9541d182ddf94d18376ec0"
14:20:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.985304 10193 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 1a1bad1ccc4b4beaa9e0dfdf3fef9185 in current term 2: Already voted for candidate 425008e39c9541d182ddf94d18376ec0 in this term.
14:20:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.991650 10167 leader_election.cc:334] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.992669 10167 leader_election.cc:334] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.993391 10252 leader_election.cc:334] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.993410 10274 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "425008e39c9541d182ddf94d18376ec0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.994338 10274 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 425008e39c9541d182ddf94d18376ec0 in current term 2: Already voted for candidate 1a1bad1ccc4b4beaa9e0dfdf3fef9185 in this term.
14:20:14.995 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:14.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:14.997757 10252 leader_election.cc:334] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:14.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.998337 10252 leader_election.cc:302] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1a1bad1ccc4b4beaa9e0dfdf3fef9185; no voters: 425008e39c9541d182ddf94d18376ec0, be7085ccc0ce4009aa50e6ac7ae00c8b
14:20:14.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:14.999262 10998 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
14:20:15.000 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.000370 10170 leader_election.cc:302] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 425008e39c9541d182ddf94d18376ec0; no voters: 1a1bad1ccc4b4beaa9e0dfdf3fef9185, be7085ccc0ce4009aa50e6ac7ae00c8b
14:20:15.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.001139 10997 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
14:20:15.015 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.016 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.035 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.036 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.037 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.037 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.075 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.076 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.076 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.077 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.078 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.115 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.117 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.118 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.119 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [9ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [24ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [28ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [44ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [45ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [65ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [103ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [142ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [182ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@463330155(state=PENDING, result=null, callback=(continuation of Deferred@1191936855 after retry RPC after error@658463361) -> (continuation of Deferred@575110688 after retry RPC after error@1467264896) -> (continuation of Deferred@1283366658 after retry RPC after error@298336094) -> (continuation of Deferred@149415225 after retry RPC after error@2057126666) -> (continuation of Deferred@1109607892 after retry RPC after error@314683393) -> (continuation of Deferred@624443725 after retry RPC after error@892191731), errback=(continuation of Deferred@1191936855 after retry RPC after error@658463361) -> (continuation of Deferred@575110688 after retry RPC after error@1467264896) -> (continuation of Deferred@1283366658 after retry RPC after error@298336094) -> (continuation of Deferred@149415225 after retry RPC after error@2057126666) -> (continuation of Deferred@1109607892 after retry RPC after error@314683393) -> (continuation of Deferred@624443725 after retry RPC after error@892191731)))
14:20:15.120 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [9ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [24ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [28ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [44ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [45ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [65ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [103ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [142ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [182ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@463330155(state=PENDING, result=null, callback=(continuation of Deferred@1191936855 after retry RPC after error@658463361) -> (continuation of Deferred@575110688 after retry RPC after error@1467264896) -> (continuation of Deferred@1283366658 after retry RPC after error@298336094) -> (continuation of Deferred@149415225 after retry RPC after error@2057126666) -> (continuation of Deferred@1109607892 after retry RPC after error@314683393) -> (continuation of Deferred@624443725 after retry RPC after error@892191731), errback=(continuation of Deferred@1191936855 after retry RPC after error@658463361) -> (continuation of Deferred@575110688 after retry RPC after error@1467264896) -> (continuation of Deferred@1283366658 after retry RPC after error@298336094) -> (continuation of Deferred@149415225 after retry RPC after error@2057126666) -> (continuation of Deferred@1109607892 after retry RPC after error@314683393) -> (continuation of Deferred@624443725 after retry RPC after error@892191731)))
14:20:15.135 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.136 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.136 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.155 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.175 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.181 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.195 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.196 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.196 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.196 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.199 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.215 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.217 [INFO - pool-46-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.219 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.256 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.260 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.276 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.277 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.277 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.286 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.288 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=151), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [24ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [45ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [46ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [61ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [122ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [124ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [142ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [144ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [150ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [151ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@764828118(state=PENDING, result=null, callback=(continuation of Deferred@908981036 after retry RPC after error@2038435607) -> (continuation of Deferred@998949037 after retry RPC after error@617445091) -> (continuation of Deferred@2107659717 after retry RPC after error@77661649) -> (continuation of Deferred@1766007360 after retry RPC after error@645282789) -> (continuation of Deferred@1502039187 after retry RPC after error@1426347519) -> (continuation of Deferred@623197245 after retry RPC after error@840667759), errback=(continuation of Deferred@908981036 after retry RPC after error@2038435607) -> (continuation of Deferred@998949037 after retry RPC after error@617445091) -> (continuation of Deferred@2107659717 after retry RPC after error@77661649) -> (continuation of Deferred@1766007360 after retry RPC after error@645282789) -> (continuation of Deferred@1502039187 after retry RPC after error@1426347519) -> (continuation of Deferred@623197245 after retry RPC after error@840667759)))
14:20:15.288 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=151), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [24ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [45ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [46ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [61ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [122ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [124ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [142ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [144ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [150ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [151ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@764828118(state=PENDING, result=null, callback=(continuation of Deferred@908981036 after retry RPC after error@2038435607) -> (continuation of Deferred@998949037 after retry RPC after error@617445091) -> (continuation of Deferred@2107659717 after retry RPC after error@77661649) -> (continuation of Deferred@1766007360 after retry RPC after error@645282789) -> (continuation of Deferred@1502039187 after retry RPC after error@1426347519) -> (continuation of Deferred@623197245 after retry RPC after error@840667759), errback=(continuation of Deferred@908981036 after retry RPC after error@2038435607) -> (continuation of Deferred@998949037 after retry RPC after error@617445091) -> (continuation of Deferred@2107659717 after retry RPC after error@77661649) -> (continuation of Deferred@1766007360 after retry RPC after error@645282789) -> (continuation of Deferred@1502039187 after retry RPC after error@1426347519) -> (continuation of Deferred@623197245 after retry RPC after error@840667759)))
14:20:15.316 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.317482 10646 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.033s from last keepalive heartbeat (effective timeout is 1.000s)
14:20:15.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.322 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.322 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.331663 10652 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.9.230.188:32935)
14:20:15.336 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.337 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.341 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.348330 10785 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.9.230.130:42049, user_credentials={real_user=slave}} blocked reactor thread for 77969.8us
14:20:15.356 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.357 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.357 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.363 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.375 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.377 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.384 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.395 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.397 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.397 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.397 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.402 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.404657 10997 raft_consensus.cc:482] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.405145 10997 raft_consensus.cc:504] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.407006 10997 leader_election.cc:288] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409), be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935)
14:20:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.411737 10274 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "425008e39c9541d182ddf94d18376ec0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" is_pre_election: true
14:20:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.412400 10274 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 425008e39c9541d182ddf94d18376ec0 in term 2.
14:20:15.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.414912 10167 leader_election.cc:334] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:15.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.415882 10170 leader_election.cc:302] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a1bad1ccc4b4beaa9e0dfdf3fef9185, 425008e39c9541d182ddf94d18376ec0; no voters: be7085ccc0ce4009aa50e6ac7ae00c8b
14:20:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.417078 10997 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:20:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.417413 10997 raft_consensus.cc:482] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.417716 10997 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 2 FOLLOWER]: Advancing to term 3
14:20:15.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.428625 10997 raft_consensus.cc:504] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:15.435 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.437 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.437 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.435834 10274 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "425008e39c9541d182ddf94d18376ec0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185"
14:20:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.436455 10274 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 2 FOLLOWER]: Advancing to term 3
14:20:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.454048 10997 leader_election.cc:288] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 3 election: Requested vote from peers 1a1bad1ccc4b4beaa9e0dfdf3fef9185 (127.9.230.189:41409), be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935)
14:20:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.458580 10274 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 425008e39c9541d182ddf94d18376ec0 in term 3.
14:20:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.464751 10170 leader_election.cc:302] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a1bad1ccc4b4beaa9e0dfdf3fef9185, 425008e39c9541d182ddf94d18376ec0; no voters:
14:20:15.467 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.467875 10167 leader_election.cc:334] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.469194 10997 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 3 FOLLOWER]: Leader election won for term 3
14:20:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.481976 10997 raft_consensus.cc:686] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [term 3 LEADER]: Becoming Leader. State: Replica: 425008e39c9541d182ddf94d18376ec0, State: Running, Role: LEADER
14:20:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.483271 10997 consensus_queue.cc:237] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } }
14:20:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.500450 11014 sys_catalog.cc:455] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 425008e39c9541d182ddf94d18376ec0. Latest consensus state: current_term: 3 leader_uuid: "425008e39c9541d182ddf94d18376ec0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.501384 11014 sys_catalog.cc:458] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: This master's current role is: LEADER
14:20:15.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.515622 11016 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
14:20:15.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.525640 11016 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=011b239f4d6c4a64b867f3f9fada3152]
14:20:15.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.527590 11016 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904]
14:20:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.534920 11016 tablet_loader.cc:96] loaded metadata for tablet 36fe95aae5bd403e8dba8bce7d5e00ea (table txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904])
14:20:15.535 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.536 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.537 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.536350 11016 tablet_loader.cc:96] loaded metadata for tablet ded3a30847e24b8cae14ffd47ff6f377 (table txn_manager_fallback [id=503d3d4b9e2148e0b48028bb1ad7f904])
14:20:15.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.537743 11016 tablet_loader.cc:96] loaded metadata for tablet e971f7d692c542f981fc9f70af9ab1d8 (table kudu_system.kudu_transactions [id=011b239f4d6c4a64b867f3f9fada3152])
14:20:15.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.539191 11016 catalog_manager.cc:1484] Initializing Kudu cluster ID...
14:20:15.543 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935; will retry
14:20:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.543767 11016 catalog_manager.cc:1259] Loaded cluster ID: 6dfa6e5a541d4958bd5614f0d7dbf206
14:20:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.544009 11016 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
14:20:15.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.548660 11016 catalog_manager.cc:1504] Loading token signing keys...
14:20:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.552878 11016 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0: Loaded TSK: 0
14:20:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.554203 11016 catalog_manager.cc:1514] Initializing in-progress tserver states...
14:20:15.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.570778 10612 heartbeater.cc:644] Failed to heartbeat to 127.9.230.188:32935 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:15.576 [DEBUG - pool-46-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [4ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [7ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [25ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [42ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [47ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [62ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [66ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [69ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [69ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [85ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [87ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [87ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [122ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [128ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [151ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [152ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [222ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, [225ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [228ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.189:41409: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.9.230.190:42571,127.9.230.189:41409,127.9.230.188:32935) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=Deferred@1615665755(state=PENDING, result=null, callback=(continuation of Deferred@2142657963 after retry RPC after error@1190957739) -> (continuation of Deferred@1268930810 after retry RPC after error@1135938583) -> (continuation of Deferred@245582543 after retry RPC after error@745076858) -> (continuation of Deferred@1968443525 after retry RPC after error@518684556) -> (continuation of Deferred@176929708 after retry RPC after error@1632315823) -> (continuation of Deferred@877583675 after retry RPC after error@2031473185) -> (continuation of Deferred@781471860 after retry RPC after error@371855960), errback=(continuation of Deferred@2142657963 after retry RPC after error@1190957739) -> (continuation of Deferred@1268930810 after retry RPC after error@1135938583) -> (continuation of Deferred@245582543 after retry RPC after error@745076858) -> (continuation of Deferred@1968443525 after retry RPC after error@518684556) -> (continuation of Deferred@176929708 after retry RPC after error@1632315823) -> (continuation of Deferred@877583675 after retry RPC after error@2031473185) -> (continuation of Deferred@781471860 after retry RPC after error@371855960)))
14:20:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.589767 10167 consensus_peers.cc:485] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 -> Peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Couldn't send request to peer be7085ccc0ce4009aa50e6ac7ae00c8b. Status: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:20:15.577 [WARN - pool-46-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:748)
14:20:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.599334 10274 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [term 3 FOLLOWER]: Refusing update from remote peer 425008e39c9541d182ddf94d18376ec0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
14:20:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.601291 11014 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
14:20:15.605 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.230.188:32935(127.9.230.188:32935)] connecting to peer
14:20:15.607 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.607 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.188:32935(127.9.230.188:32935)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.607 [INFO - pool-46-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.230.188:32935: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935
14:20:15.607 [DEBUG - pool-46-thread-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:20:15.608 [DEBUG - pool-46-thread-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=12), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.190:42571, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.190:42571: OK, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.189:41409, [9ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.230.188:32935, [12ms] Sub RPC ConnectToMaster: received response from server master-127.9.230.188:32935: Network error: Failed to connect to peer master-127.9.230.188:32935(127.9.230.188:32935): Connection refused: /127.9.230.188:32935, deferred=null) after lookup
14:20:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.640372 11008 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
14:20:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.648581 11020 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 425008e39c9541d182ddf94d18376ec0. Latest consensus state: current_term: 3 leader_uuid: "425008e39c9541d182ddf94d18376ec0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.649755 11020 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: This master's current role is: FOLLOWER
14:20:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.681562 10909 heartbeater.cc:644] Failed to heartbeat to 127.9.230.188:32935 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111)
14:20:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.685995 11020 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "425008e39c9541d182ddf94d18376ec0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.686830 11020 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a1bad1ccc4b4beaa9e0dfdf3fef9185 [sys.catalog]: This master's current role is: FOLLOWER
14:20:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.699996 10997 sys_catalog.cc:455] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "425008e39c9541d182ddf94d18376ec0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "425008e39c9541d182ddf94d18376ec0" member_type: VOTER last_known_addr { host: "127.9.230.190" port: 42571 } } peers { permanent_uuid: "1a1bad1ccc4b4beaa9e0dfdf3fef9185" member_type: VOTER last_known_addr { host: "127.9.230.189" port: 41409 } } peers { permanent_uuid: "be7085ccc0ce4009aa50e6ac7ae00c8b" member_type: VOTER last_known_addr { host: "127.9.230.188" port: 32935 } } }
14:20:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.700713 10997 sys_catalog.cc:458] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 [sys.catalog]: This master's current role is: LEADER
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:15.796006 10183 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45574:
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:20:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:20:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:20:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.800101 10183 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:20:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.829902 10213 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:45550 (request call id 29) took 220 ms (client timeout 238 ms). Trace:
14:20:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:15.830246 10213 rpcz_store.cc:269] 0501 14:20:15.609018 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:20:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:15.609224 (+ 206us) service_pool.cc:225] Handling call
14:20:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 14:20:15.829707 (+220483us) inbound_call.cc:173] Queueing success response
14:20:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":972,"init.queue_time_us":14143,"init.run_cpu_time_us":27847,"init.run_wall_time_us":185833,"thread_start_us":101203,"threads_started":5}
14:20:15.832 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1714573214 last_transition_timestamp: 1714573215
14:20:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:16.064049 10167 consensus_peers.cc:485] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 -> Peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Couldn't send request to peer be7085ccc0ce4009aa50e6ac7ae00c8b. Status: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:20:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 14:20:16.562527 10167 consensus_peers.cc:485] T 00000000000000000000000000000000 P 425008e39c9541d182ddf94d18376ec0 -> Peer be7085ccc0ce4009aa50e6ac7ae00c8b (127.9.230.188:32935): Couldn't send request to peer be7085ccc0ce4009aa50e6ac7ae00c8b. Status: Network error: Client connection negotiation failed: client connection to 127.9.230.188:32935: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:20:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:16.700114 10911 heartbeater.cc:503] Master 127.9.230.190:42571 was elected leader, sending a full tablet report...
14:20:16.724 [DEBUG - main] (Connection.java:575) [peer master-127.9.230.190:42571(127.9.230.190:42571)] disconnecting while in state READY
14:20:16.724 [DEBUG - main] (Connection.java:575) [peer master-127.9.230.189:41409(127.9.230.189:41409)] disconnecting while in state READY
14:20:16.724 [DEBUG - main] (Connection.java:575) [peer f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)] disconnecting while in state READY
14:20:16.724 [DEBUG - main] (Connection.java:575) [peer master-127.9.230.188:32935(127.9.230.188:32935)] disconnecting while in state TERMINATED
14:20:16.725 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.230.190:42571(127.9.230.190:42571)] handling channelInactive
14:20:16.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.230.190:42571(127.9.230.190:42571)] cleaning up while in state READY due to: connection closed
14:20:16.726 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:16.726 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.230.189:41409(127.9.230.189:41409)] handling channelInactive
14:20:16.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.230.189:41409(127.9.230.189:41409)] cleaning up while in state READY due to: connection closed
14:20:16.727 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)] handling channelInactive
14:20:16.727 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f785f90643644d76b1f59b3b3907141d(127.9.230.131:35983)] cleaning up while in state READY due to: connection closed
14:20:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:16.726833 10138 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdRM744/build/tsan/bin/kudu with pid 10448
14:20:16.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:16.754247 10138 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdRM744/build/tsan/bin/kudu with pid 10617
14:20:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:16.791035 10138 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdRM744/build/tsan/bin/kudu with pid 10767
14:20:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:16.821364 10138 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdRM744/build/tsan/bin/kudu with pid 10150
14:20:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 14:20:16.852381 10138 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdRM744/build/tsan/bin/kudu with pid 10230
14:20:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T14:20:17Z chronyd exiting
14:20:18.982 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1714573214 last_transition_timestamp: 1714573215
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 3aa5299cc1ff47f1b7cd39baadf9db57 (127.9.230.130:42049): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1714573214 last_transition_timestamp: 1714573215
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more