15:05:37.289 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:05:37.302 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskdK4Kxd/test-tmp
15:05:37.305 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:05:37.308 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskdK4Kxd/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:05:37.580 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914"
extra_tserver_flags: "--flush_threshold_mb=1"
extra_tserver_flags: "--flush_threshold_secs=1"
extra_tserver_flags: "--enable_rowset_compaction=false"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:05:37.729 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:05:37.730 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:05:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.734985 10614 mini_oidc.cc:150] Starting JWKS server
15:05:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.739168 10614 webserver.cc:458] Webserver started at http://127.0.0.1:42965/ using document root <none> and password file <none>
15:05:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.739753 10614 mini_oidc.cc:186] Starting OIDC Discovery server
15:05:37.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.741914 10614 webserver.cc:458] Webserver started at http://127.0.0.1:33477/ using document root <none> and password file <none>
15:05:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:37Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:05:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:37Z Disabled control of system clock
15:05:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.809552 10614 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu
15:05:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu
15:05:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal
15:05:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/logs
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data/info.pb
15:05:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.93.190:35649
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.93.190
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.93.148:34863
15:05:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.93.190:35649,127.10.93.189:37541,127.10.93.188:46835
15:05:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42965/jwks.json with env {}
15:05:38.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.122731 10626 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:38.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.123313 10626 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:38.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.123800 10626 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.158421 10626 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.158851 10626 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.159210 10626 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.159540 10626 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.159857 10626 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.160055 10626 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42965/jwks.json
15:05:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.196779 10626 master_runner.cc:386] Master server non-default flags:
15:05:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.93.148:34863
15:05:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.93.190:35649,127.10.93.189:37541,127.10.93.188:46835
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.93.190:35649
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42965/jwks.json
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data/info.pb
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.93.190
15:05:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/logs
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.198441 10626 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.200378 10626 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.217206 10632 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.218464 10633 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.219689 10626 server_base.cc:1034] running on GCE node
15:05:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.219573 10635 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.317768 10626 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.320703 10626 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.322166 10626 hybrid_clock.cc:648] HybridClock initialized: now 1736780739322118 us; error 51 us; skew 500 ppm
15:05:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.323164 10626 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.329830 10626 webserver.cc:458] Webserver started at http://127.10.93.190:46159/ using document root <none> and password file <none>
15:05:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.331046 10626 fs_manager.cc:362] Metadata directory not provided
15:05:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.331290 10626 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.331854 10626 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.337219 10626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data/instance:
15:05:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d9470b98d314827813ac12c27b21384"
15:05:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-25rk"
15:05:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.338691 10626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal/instance:
15:05:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d9470b98d314827813ac12c27b21384"
15:05:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-25rk"
15:05:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.348780 10626 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
15:05:39.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.355414 10642 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:39.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.356860 10626 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:05:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.357275 10626 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data,/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal
15:05:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d9470b98d314827813ac12c27b21384"
15:05:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-25rk"
15:05:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.357743 10626 fs_report.cc:389] FS layout report
15:05:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal
15:05:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal
15:05:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data/data
15:05:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.407750 10626 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:39.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.409380 10626 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:39.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.409920 10626 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.481838 10626 rpc_server.cc:307] RPC server started. Bound to: 127.10.93.190:35649
15:05:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.481979 10693 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.93.190:35649 every 8 connection(s)
15:05:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.484870 10626 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/data/info.pb
15:05:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.488811 10614 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu as pid 10626
15:05:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.489377 10614 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-0/wal/instance
15:05:39.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.491786 10694 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.494792 10614 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/logs
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data/info.pb
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.93.189:37541
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.93.189
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.93.148:34863
15:05:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.93.190:35649,127.10.93.189:37541,127.10.93.188:46835
15:05:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42965/jwks.json with env {}
15:05:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.506284 10694 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } has no permanent_uuid. Determining permanent_uuid...
15:05:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.531975 10694 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } has no permanent_uuid. Determining permanent_uuid...
15:05:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.534646 10646 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.539045 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:39.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.588675 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 1
15:05:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.593025 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.661586 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 2
15:05:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.665769 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.757357 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 3
15:05:39.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.762076 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:39.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.814316 10696 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:39.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.814891 10696 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:39.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.815436 10696 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:39.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.847131 10696 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:39.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.847426 10696 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:39.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.847631 10696 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:39.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.847815 10696 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.848004 10696 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.848150 10696 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42965/jwks.json
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.883800 10696 master_runner.cc:386] Master server non-default flags:
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.93.148:34863
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.93.190:35649,127.10.93.189:37541,127.10.93.188:46835
15:05:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.93.189:37541
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42965/jwks.json
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data/info.pb
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.93.189
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/logs
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:39.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:39.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.885257 10696 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.887012 10696 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.903492 10705 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.905484 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 4
15:05:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.909416 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.208842 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 5
15:05:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.213934 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.761417 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 6
15:05:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.768739 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.189:37541: Network error: Client connection negotiation failed: client connection to 127.10.93.189:37541: connect: Connection refused (error 111)
15:05:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.904022 10706 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.905243 10696 server_base.cc:1034] running on GCE node
15:05:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.904772 10708 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.010514 10696 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.014226 10696 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:41.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.015619 10696 hybrid_clock.cc:648] HybridClock initialized: now 1736780741015568 us; error 49 us; skew 500 ppm
15:05:41.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.016546 10696 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:41.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.023270 10696 webserver.cc:458] Webserver started at http://127.10.93.189:38971/ using document root <none> and password file <none>
15:05:41.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.024410 10696 fs_manager.cc:362] Metadata directory not provided
15:05:41.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.024742 10696 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.025838 10696 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:41.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.031167 10696 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data/instance:
15:05:41.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9953dddaf436467baf8b8309e3fda337"
15:05:41.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-25rk"
15:05:41.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.032490 10696 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal/instance:
15:05:41.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9953dddaf436467baf8b8309e3fda337"
15:05:41.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-25rk"
15:05:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.039980 10696 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
15:05:41.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.046392 10717 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.047487 10696 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:05:41.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.047839 10696 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data,/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal
15:05:41.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9953dddaf436467baf8b8309e3fda337"
15:05:41.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-25rk"
15:05:41.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.048223 10696 fs_report.cc:389] FS layout report
15:05:41.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal
15:05:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal
15:05:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data/data
15:05:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.103072 10696 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.104619 10696 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.105113 10696 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.176246 10696 rpc_server.cc:307] RPC server started. Bound to: 127.10.93.189:37541
15:05:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.176316 10768 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.93.189:37541 every 8 connection(s)
15:05:41.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.179056 10696 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/data/info.pb
15:05:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.184491 10614 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu as pid 10696
15:05:41.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.184881 10614 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-1/wal/instance
15:05:41.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.185317 10769 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.187171 10614 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu
15:05:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu
15:05:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal
15:05:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data
15:05:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/logs
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data/info.pb
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.93.188:46835
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.93.188
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.93.148:34863
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.93.190:35649,127.10.93.189:37541,127.10.93.188:46835
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42965/jwks.json with env {}
15:05:41.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.206615 10769 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.237717 10769 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.253052 10769 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.256567 10721 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.262472 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.311950 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 1
15:05:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.316747 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.385179 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 2
15:05:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.389304 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.480753 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 3
15:05:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.486369 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.599421 10771 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.600128 10771 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.600858 10771 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:41.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.629848 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 4
15:05:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.634404 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.634795 10771 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.635031 10771 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.635315 10771 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.635567 10771 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.635749 10771 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.635895 10771 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42965/jwks.json
15:05:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.677104 10771 master_runner.cc:386] Master server non-default flags:
15:05:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.93.148:34863
15:05:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.93.190:35649,127.10.93.189:37541,127.10.93.188:46835
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.93.188:46835
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42965/jwks.json
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data/info.pb
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.93.188
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/logs
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.678699 10771 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:41.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.680763 10771 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.700544 10781 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.829433 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } attempt: 7
15:05:41.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.848120 10694 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.854367 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.912897 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 1
15:05:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.917412 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:41.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.933919 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 5
15:05:41.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.939036 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:42.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.999058 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 2
15:05:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.004036 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.089797 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 3
15:05:42.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.095186 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.235735 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 4
15:05:42.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.244503 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.537678 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 5
15:05:42.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.546904 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:42.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.706568 10784 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.709856 10771 server_base.cc:1034] running on GCE node
15:05:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.701673 10782 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:42.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.950634 10771 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.584546 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 6
15:05:44.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.618803 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 6
15:05:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.642434 10646 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111) [suppressed 12 similar messages]
15:05:44.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.644837 10771 thread.cc:635] ntp client (ntp) Time spent creating pthread: real 1.692s user 1.187s sys 0.490s
15:05:44.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.645660 10771 thread.cc:601] ntp client (ntp) Time spent starting thread: real 1.693s user 1.187s sys 0.490s
15:05:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.647481 10771 hybrid_clock.cc:648] HybridClock initialized: now 1736780744647424 us; error 49 us; skew 500 ppm
15:05:44.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.648831 10771 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.653199 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:44.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.671334 10771 webserver.cc:458] Webserver started at http://127.10.93.188:38867/ using document root <none> and password file <none>
15:05:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.672015 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.672955 10771 fs_manager.cc:362] Metadata directory not provided
15:05:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.673621 10771 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.674294 10771 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.683421 10771 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data/instance:
15:05:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e808032d5f9b48fcb99561916c29eccf"
15:05:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:44 on dist-test-slave-25rk"
15:05:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.685153 10771 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal/instance:
15:05:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e808032d5f9b48fcb99561916c29eccf"
15:05:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:44 on dist-test-slave-25rk"
15:05:44.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.698395 10771 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.009s sys 0.002s
15:05:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.707224 10798 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.709023 10771 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.709477 10771 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data,/tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e808032d5f9b48fcb99561916c29eccf"
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:44 on dist-test-slave-25rk"
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.710011 10771 fs_report.cc:389] FS layout report
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal
15:05:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data/data
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.929158 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 7
15:05:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.933418 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 7
15:05:45.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.949849 10771 thread.cc:635] rpc reactor (reactor) Time spent creating pthread: real 1.156s user 0.650s sys 0.395s
15:05:45.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.951540 10771 thread.cc:601] rpc reactor (reactor) Time spent starting thread: real 1.158s user 0.651s sys 0.395s
15:05:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.958834 10694 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:45.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.966033 10769 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.93.188:46835: Network error: Client connection negotiation failed: client connection to 127.10.93.188:46835: connect: Connection refused (error 111)
15:05:45.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.972285 10771 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:45.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.974706 10771 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:45.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.975217 10771 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.118515 10771 thread.cc:601] excess-log-deleter (server) Time spent starting thread: real 0.994s user 0.403s sys 0.589s
15:05:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.122620 10771 rpc_server.cc:307] RPC server started. Bound to: 127.10.93.188:46835
15:05:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.126868 10771 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/data/info.pb
15:05:47.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.129815 10853 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.93.188:46835 every 8 connection(s)
15:05:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.135138 10614 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskdK4Kxd/build/tsan/bin/kudu as pid 10771
15:05:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.135679 10614 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskdK4Kxd/test-tmp/mini-kudu-cluster1058165397072388914/master-2/wal/instance
15:05:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.150069 10854 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:47.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.176120 10854 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } has no permanent_uuid. Determining permanent_uuid...
15:05:47.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.205943 10854 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } has no permanent_uuid. Determining permanent_uuid...
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.219851 10854 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } has no permanent_uuid. Determining permanent_uuid...
15:05:47.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.240447 10854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: Bootstrap starting.
15:05:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.247915 10854 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: Neither blocks nor log segments found. Creating new log.
15:05:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.249787 10854 log.cc:826] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: Log is configured to *not* fsync() on all Append() calls
15:05:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.255503 10854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: No bootstrap required, opened a new log
15:05:47.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.280639 10854 raft_consensus.cc:357] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:47.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.281360 10854 raft_consensus.cc:383] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:47.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.281571 10854 raft_consensus.cc:738] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e808032d5f9b48fcb99561916c29eccf, State: Initialized, Role: FOLLOWER
15:05:47.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.282353 10854 consensus_queue.cc:260] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.285226 10862 sys_catalog.cc:455] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.285921 10862 sys_catalog.cc:458] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: This master's current role is: FOLLOWER
15:05:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.286895 10854 sys_catalog.cc:564] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: configured and running, proceeding with master startup.
15:05:47.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.319311 10873 catalog_manager.cc:1559] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.319716 10873 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:47.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.312275 10771 master_runner.cc:186] Error getting master registration for 127.10.93.190:35649: OK, instance_id { permanent_uuid: "2d9470b98d314827813ac12c27b21384" instance_seqno: 1736780739461644 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:47.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.345305 10771 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.360110 10808 tablet.cc:2367] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:47.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.850973 10623 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.ConnectToMaster -> {remote=127.10.93.189:37541, user_credentials={real_user=slave}} blocked reactor thread for 342406us
15:05:47.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.856359 10876 raft_consensus.cc:491] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:47.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.856853 10876 raft_consensus.cc:513] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:47.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.870226 10876 leader_election.cc:290] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2d9470b98d314827813ac12c27b21384 (127.10.93.190:35649), 9953dddaf436467baf8b8309e3fda337 (127.10.93.189:37541)
15:05:48.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.020356 10694 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 8
15:05:48.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.056481 10769 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } attempt: 8
15:05:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.656163 10873 catalog_manager.cc:1559] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.663811 10873 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.731885 10769 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: Bootstrap starting.
15:05:48.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.742072 10769 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: Neither blocks nor log segments found. Creating new log.
15:05:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.745937 10769 log.cc:826] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: Log is configured to *not* fsync() on all Append() calls
15:05:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.746083 10694 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384: Bootstrap starting.
15:05:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.779178 10694 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384: Neither blocks nor log segments found. Creating new log.
15:05:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.780895 10769 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: No bootstrap required, opened a new log
15:05:48.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.782920 10694 log.cc:826] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384: Log is configured to *not* fsync() on all Append() calls
15:05:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.802153 10694 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384: No bootstrap required, opened a new log
15:05:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.822301 10769 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.823632 10769 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.824193 10769 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9953dddaf436467baf8b8309e3fda337, State: Initialized, Role: FOLLOWER
15:05:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.825390 10769 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.306049 10800 connection.cc:232] RPC call timeout handler was delayed by -1.0030271789999006s: this may be due to a process-wide pause such as swapping, logging-related delays, or allocator lock contention. Will allow extra 0.16010000000000002s for a response
15:05:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.308614 10723 tablet.cc:2367] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.316102 10769 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: configured and running, proceeding with master startup.
15:05:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.318205 10744 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e808032d5f9b48fcb99561916c29eccf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9953dddaf436467baf8b8309e3fda337" is_pre_election: true
15:05:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.319108 10744 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e808032d5f9b48fcb99561916c29eccf in term 0.
15:05:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.319715 10744 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:55246 (request call id 1) took 2452 ms (client timeout 1601 ms). Trace:
15:05:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.320065 10744 rpcz_store.cc:269] 0113 15:05:47.867721 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:47.867894 (+ 173us) service_pool.cc:225] Handling call
15:05:50.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.319635 (+2451741us) inbound_call.cc:173] Queueing success response
15:05:50.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:05:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.326135 10804 connection.cc:232] RPC call timeout handler was delayed by -1.0076972489999889s: this may be due to a process-wide pause such as swapping, logging-related delays, or allocator lock contention. Will allow extra 0.16010000000000002s for a response
15:05:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.327133 10804 leader_election.cc:304] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9953dddaf436467baf8b8309e3fda337, e808032d5f9b48fcb99561916c29eccf; no voters:
15:05:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.832342 10884 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:50.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.334435 10884 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.335695 10873 catalog_manager.cc:1559] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.336026 10873 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.338747 10888 raft_consensus.cc:2798] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Leader pre-election won for term 1
15:05:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.339125 10888 raft_consensus.cc:491] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:05:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.339449 10888 raft_consensus.cc:3054] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 0 FOLLOWER]: Advancing to term 1
15:05:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.345667 10888 raft_consensus.cc:513] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:50.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.354619 10888 leader_election.cc:290] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [CANDIDATE]: Term 1 election: Requested vote from peers 2d9470b98d314827813ac12c27b21384 (127.10.93.190:35649), 9953dddaf436467baf8b8309e3fda337 (127.10.93.189:37541)
15:05:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.376061 10744 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e808032d5f9b48fcb99561916c29eccf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9953dddaf436467baf8b8309e3fda337"
15:05:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.377643 10744 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 0 FOLLOWER]: Advancing to term 1
15:05:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.381031 10694 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.382081 10694 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.382428 10694 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d9470b98d314827813ac12c27b21384, State: Initialized, Role: FOLLOWER
15:05:50.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.394136 10744 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e808032d5f9b48fcb99561916c29eccf in term 1.
15:05:50.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.397972 10804 leader_election.cc:304] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9953dddaf436467baf8b8309e3fda337, e808032d5f9b48fcb99561916c29eccf; no voters:
15:05:50.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.398936 10888 raft_consensus.cc:2798] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 1 FOLLOWER]: Leader election won for term 1
15:05:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.399349 10694 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.402577 10888 raft_consensus.cc:695] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 1 LEADER]: Becoming Leader. State: Replica: e808032d5f9b48fcb99561916c29eccf, State: Running, Role: LEADER
15:05:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.405577 10888 consensus_queue.cc:237] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:50.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.409637 10892 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:50.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.410696 10892 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.417062 10669 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e808032d5f9b48fcb99561916c29eccf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d9470b98d314827813ac12c27b21384" is_pre_election: true
15:05:50.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.419265 10669 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e808032d5f9b48fcb99561916c29eccf in term 0.
15:05:50.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.420094 10669 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:36202 (request call id 2) took 2550 ms (client timeout 1601 ms). Trace:
15:05:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.420480 10669 rpcz_store.cc:269] 0113 15:05:47.870044 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:47.870227 (+ 183us) service_pool.cc:225] Handling call
15:05:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.419952 (+2549725us) inbound_call.cc:173] Queueing success response
15:05:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:05:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.417944 10668 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e808032d5f9b48fcb99561916c29eccf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d9470b98d314827813ac12c27b21384"
15:05:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.421921 10668 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 0 FOLLOWER]: Advancing to term 1
15:05:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.429194 10668 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e808032d5f9b48fcb99561916c29eccf in term 1.
15:05:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.430001 10694 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [sys.catalog]: configured and running, proceeding with master startup.
15:05:50.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.443428 10891 sys_catalog.cc:455] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: SysCatalogTable state changed. Reason: New leader e808032d5f9b48fcb99561916c29eccf. Latest consensus state: current_term: 1 leader_uuid: "e808032d5f9b48fcb99561916c29eccf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.444850 10891 sys_catalog.cc:458] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: This master's current role is: LEADER
15:05:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.452804 10901 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:05:50.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.461527 10901 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:05:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.462641 10904 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:05:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.497499 10744 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 1 FOLLOWER]: Refusing update from remote peer e808032d5f9b48fcb99561916c29eccf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.499679 10668 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Refusing update from remote peer e808032d5f9b48fcb99561916c29eccf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.499557 10891 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.507059 10891 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.004s
15:05:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.467855 10696 master_runner.cc:186] Error getting master registration for 127.10.93.190:35649: OK, instance_id { permanent_uuid: "2d9470b98d314827813ac12c27b21384" instance_seqno: 1736780739461644 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.519263 10696 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.569985 10906 mvcc.cc:204] Tried to move back new op lower bound from 7113853953996181504 to 7113853953705783296. Current Snapshot: MvccSnapshot[applied={T|T < 7113853953996181504}]
15:05:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.582816 10884 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e808032d5f9b48fcb99561916c29eccf. Latest consensus state: current_term: 1 leader_uuid: "e808032d5f9b48fcb99561916c29eccf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:50.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.584566 10884 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.594234 10891 sys_catalog.cc:455] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e808032d5f9b48fcb99561916c29eccf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:50.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.595197 10891 sys_catalog.cc:458] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: This master's current role is: LEADER
15:05:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.684249 10648 tablet.cc:2367] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.694403 10668 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:36202 (request call id 5) took 2185 ms. Trace:
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.694787 10668 rpcz_store.cc:276] 0113 15:05:50.509249 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.509393 (+ 144us) service_pool.cc:225] Handling call
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.510813 (+ 1420us) raft_consensus.cc:1466] Updating replica for 2 ops
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.511830 (+ 1017us) raft_consensus.cc:1529] Early marking committed up to index 0
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.511885 (+ 55us) raft_consensus.cc:1534] Triggering prepare for 2 ops
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.513030 (+ 1145us) write_op.cc:270] Start()
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.513566 (+ 536us) write_op.cc:276] Timestamp: P: 1736780750487349 usec, L: 0
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.553977 (+ 40411us) log.cc:844] Serialized 255 byte log entry
15:05:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.586588 (+ 32611us) raft_consensus.cc:1640] Marking committed up to 0
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.586644 (+ 56us) raft_consensus.cc:1692] Filling consensus response to leader.
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.587586 (+ 942us) raft_consensus.cc:1663] Waiting on the replicates to finish logging
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:52.694113 (+2106527us) raft_consensus.cc:1680] finished
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:52.694143 (+ 30us) raft_consensus.cc:1686] UpdateReplicas() finished
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:52.694308 (+ 165us) inbound_call.cc:173] Queueing success response
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Related trace 'op':
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.551366 (+ 0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.557945 (+ 6579us) write_op.cc:432] Acquiring schema lock in shared mode
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.558018 (+ 73us) write_op.cc:435] Acquired schema lock
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.558049 (+ 31us) tablet.cc:659] Decoding operations
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.558906 (+ 857us) write_op.cc:620] Acquiring the partition lock for write op
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.559117 (+ 211us) write_op.cc:641] Partition lock acquired for write op
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.559184 (+ 67us) tablet.cc:682] Acquiring locks for 1 operations
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.560202 (+ 1018us) tablet.cc:698] Row locks acquired
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.560234 (+ 32us) write_op.cc:260] PREPARE: finished
15:05:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"raft.queue_time_us":87,"spinlock_wait_cycles":173201152,"thread_start_us":31799,"threads_started":1,"wal-append.queue_time_us":29016,"child_traces":[["op",{"num_ops":1,"partition_lock_wait_us":103,"prepare.queue_time_us":19823,"prepare.run_cpu_time_us":4318,"prepare.run_wall_time_us":9257,"thread_start_us":19760,"threads_started":1}]]}
15:05:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.706490 10884 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e808032d5f9b48fcb99561916c29eccf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.707203 10884 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [sys.catalog]: This master's current role is: FOLLOWER
15:05:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.707684 10884 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e808032d5f9b48fcb99561916c29eccf)
15:05:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.708076 10884 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:52.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.710963 10892 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e808032d5f9b48fcb99561916c29eccf. Latest consensus state: current_term: 1 leader_uuid: "e808032d5f9b48fcb99561916c29eccf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.711766 10892 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [sys.catalog]: This master's current role is: FOLLOWER
15:05:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.712350 10892 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e808032d5f9b48fcb99561916c29eccf)
15:05:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.712790 10892 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:52.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.715409 10892 leader_election.cc:290] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9953dddaf436467baf8b8309e3fda337 (127.10.93.189:37541), e808032d5f9b48fcb99561916c29eccf (127.10.93.188:46835)
15:05:52.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.722295 10744 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d9470b98d314827813ac12c27b21384" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "9953dddaf436467baf8b8309e3fda337" is_pre_election: true
15:05:52.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.722935 10744 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d9470b98d314827813ac12c27b21384 in term 1.
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.723922 10884 leader_election.cc:290] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2d9470b98d314827813ac12c27b21384 (127.10.93.190:35649), e808032d5f9b48fcb99561916c29eccf (127.10.93.188:46835)
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.724366 10646 leader_election.cc:304] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2d9470b98d314827813ac12c27b21384, 9953dddaf436467baf8b8309e3fda337; no voters:
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.726305 10829 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d9470b98d314827813ac12c27b21384" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "e808032d5f9b48fcb99561916c29eccf" is_pre_election: true
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.727411 10901 catalog_manager.cc:1348] Generated new cluster ID: 5ca0902679e54fa3996ccee78f761959
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.727818 10901 catalog_manager.cc:1486] Time spent T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf: Initializing Kudu cluster ID: real 2.266s user 0.007s sys 0.008s
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.728160 10901 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:05:52.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.730774 10922 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:05:52.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.736085 10829 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9953dddaf436467baf8b8309e3fda337" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "e808032d5f9b48fcb99561916c29eccf" is_pre_election: true
15:05:52.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.738338 10914 log.cc:926] Time spent T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: Append to log took a long time: real 2.093s user 1.386s sys 0.697s
15:05:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.744309 10668 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9953dddaf436467baf8b8309e3fda337" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "2d9470b98d314827813ac12c27b21384" is_pre_election: true
15:05:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.744979 10921 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:05:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.745337 10921 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Starting leader election (detected failure of leader e808032d5f9b48fcb99561916c29eccf)
15:05:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.745649 10921 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 1 FOLLOWER]: Advancing to term 2
15:05:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.746577 10719 leader_election.cc:304] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9953dddaf436467baf8b8309e3fda337; no voters: 2d9470b98d314827813ac12c27b21384, e808032d5f9b48fcb99561916c29eccf
15:05:52.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.740475 10734 rpcz_store.cc:267] Call kudu.master.MasterService.ConnectToMaster from 127.0.0.1:55232 (request call id 33) took 2214 ms (client timeout 1615 ms). Trace:
15:05:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.750190 10923 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.751786 10734 rpcz_store.cc:269] 0113 15:05:50.525959 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.526134 (+ 175us) service_pool.cc:225] Handling call
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:50.587729 (+ 61595us) spinlock_profiling.cc:243] Waited 26.1 ms on lock 0x7b54000415d4. stack: 00007f7b2dc60853 00007f7b2dc60750 00007f7b2d7d5584 00000000004fcd24 00000000004fccb9 00007f7b34b60d50 00007f7b34be3f2a 00007f7b36888dec 00007f7b36984009 00007f7b346e1544 00007f7b346e14d2 00007f7b346e1401 00007f7b346e137c 00007f7b346e0632 00007f7b329a8dfc 00007f7b329a8236
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:52.739985 (+2152256us) spinlock_profiling.cc:243] Waited 13.5 ms on lock 0x7b54000415d4. stack: 00007f7b2dc60853 00007f7b2dc60750 00007f7b2d7d5584 00000000004fcd24 00000000004fccb9 00007f7b34b606d1 00007f7b34be3fe9 00007f7b36945c66 00007f7b36984137 00007f7b346e1544 00007f7b346e14d2 00007f7b346e1401 00007f7b346e137c 00007f7b346e0632 00007f7b329a8dfc 00007f7b329a8236
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:52.740284 (+ 299us) inbound_call.cc:173] Queueing success response
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":91076608}
15:05:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.753314 10921 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:52.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.756902 10744 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d9470b98d314827813ac12c27b21384" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "9953dddaf436467baf8b8309e3fda337"
15:05:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.760349 10829 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2d9470b98d314827813ac12c27b21384" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "e808032d5f9b48fcb99561916c29eccf"
15:05:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.762044 10904 catalog_manager.cc:1260] Loaded cluster ID: 5ca0902679e54fa3996ccee78f761959
15:05:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.762297 10904 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: loading cluster ID for follower catalog manager: success
15:05:52.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.764696 10646 leader_election.cc:304] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2d9470b98d314827813ac12c27b21384; no voters: 9953dddaf436467baf8b8309e3fda337, e808032d5f9b48fcb99561916c29eccf
15:05:52.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.766602 10904 catalog_manager.cc:1577] T 00000000000000000000000000000000 P 9953dddaf436467baf8b8309e3fda337: acquiring CA information for follower catalog manager: Not found: root CA entry not found
15:05:52.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:52.766893 10904 catalog_manager.cc:874] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
15:05:52.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.767346 10892 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
15:05:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.769611 10771 master_runner.cc:232] Existing masters have differing terms: 2 vs 1
15:05:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.769726 10668 raft_consensus.cc:1235] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 2 FOLLOWER]: Rejecting Update request from peer e808032d5f9b48fcb99561916c29eccf for earlier term 1. Current term is 2. Ops: []
15:05:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.770043 10771 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.764897 10921 leader_election.cc:290] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [CANDIDATE]: Term 2 election: Requested vote from peers 9953dddaf436467baf8b8309e3fda337 (127.10.93.189:37541), e808032d5f9b48fcb99561916c29eccf (127.10.93.188:46835)
15:05:52.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.775341 10925 consensus_queue.cc:1046] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 }, Status: INVALID_TERM, Last received: 1.2, Next index: 3, Last known committed idx: 0, Time since last communication: 0.001s
15:05:52.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) F20250113 15:05:52.800763 10614 external_mini_cluster.cc:638] Check failed: leader_master() != nullptr Must have started at least 1 master before adding tablet servers
15:05:52.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Check failure stack trace: ***
15:05:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.807722 10888 sys_catalog.cc:455] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e808032d5f9b48fcb99561916c29eccf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } } }
15:05:52.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.808568 10888 sys_catalog.cc:458] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [sys.catalog]: This master's current role is: LEADER
15:05:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.817783 10668 raft_consensus.cc:1235] T 00000000000000000000000000000000 P 2d9470b98d314827813ac12c27b21384 [term 2 FOLLOWER]: Rejecting Update request from peer e808032d5f9b48fcb99561916c29eccf for earlier term 1. Current term is 2. Ops: [1.3-1.3]
15:05:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.821832 10928 raft_consensus.cc:3049] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 1 LEADER]: Stepping down as leader of term 1
15:05:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.822863 10928 raft_consensus.cc:738] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e808032d5f9b48fcb99561916c29eccf, State: Running, Role: LEADER
15:05:52.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.823721 10888 consensus_queue.cc:1046] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 }, Status: INVALID_TERM, Last received: 1.2, Next index: 3, Last known committed idx: 0, Time since last communication: 0.000s
15:05:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.827800 10928 consensus_queue.cc:260] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2d9470b98d314827813ac12c27b21384" member_type: VOTER last_known_addr { host: "127.10.93.190" port: 35649 } } peers { permanent_uuid: "9953dddaf436467baf8b8309e3fda337" member_type: VOTER last_known_addr { host: "127.10.93.189" port: 37541 } } peers { permanent_uuid: "e808032d5f9b48fcb99561916c29eccf" member_type: VOTER last_known_addr { host: "127.10.93.188" port: 46835 } }
15:05:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:52.829308 10928 raft_consensus.cc:3054] T 00000000000000000000000000000000 P e808032d5f9b48fcb99561916c29eccf [term 1 FOLLOWER]: Advancing to term 2
15:05:52.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1304ff1950 google::LogMessage::Flush()
15:05:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1304ff6a8b google::LogMessageFatal::~LogMessageFatal()
15:05:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f13103d3c1d kudu::cluster::ExternalMiniCluster::AddTabletServer()
15:05:52.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f13103d1c9b kudu::cluster::ExternalMiniCluster::Start()
15:05:52.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6510aa kudu::tools::(anonymous namespace)::ProcessRequest()
15:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64f9a5 kudu::tools::(anonymous namespace)::RunControlShell()
15:05:52.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x529a02 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
15:05:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x52996d std::__1::__invoke_void_return_wrapper<>::__call<>()
15:05:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x5298dd std::__1::__function::__alloc_func<>::operator()()
15:05:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x528546 std::__1::__function::__func<>::operator()()
15:05:52.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1310144f90 std::__1::__function::__value_func<>::operator()()
15:05:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f13101377f2 std::__1::function<>::operator()()
15:05:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1310135583 kudu::tools::Action::Run()
15:05:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6455a1 kudu::tools::DispatchCommand()
15:05:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x645ed8 kudu::tools::RunTool()
15:05:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64641f main
15:05:52.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1303b96bf7 __libc_start_main
15:05:52.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x43399a _start
15:05:53.603 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSplitKeyRange.testSplitKeyRange: failed attempt 1
java.io.EOFException: null
at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_275]
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)