15:05:36.181 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:05:36.195 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task7o5v58/test-tmp
15:05:36.198 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:05:36.202 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task7o5v58/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:05:36.466 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:05:36.653 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:05:36.654 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:05:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.658946 4168 mini_oidc.cc:150] Starting JWKS server
15:05:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.662945 4168 webserver.cc:458] Webserver started at http://127.0.0.1:44571/ using document root <none> and password file <none>
15:05:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.663442 4168 mini_oidc.cc:186] Starting OIDC Discovery server
15:05:36.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.665467 4168 webserver.cc:458] Webserver started at http://127.0.0.1:43871/ using document root <none> and password file <none>
15:05:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:36Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:05:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:36Z Disabled control of system clock
15:05:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.734814 4168 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7o5v58/build/tsan/bin/kudu
15:05:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7o5v58/build/tsan/bin/kudu
15:05:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal
15:05:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/logs
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data/info.pb
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.18.62:43303
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.18.62
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.18.20:44143
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.18.62:43303,127.4.18.61:38117,127.4.18.60:42413
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44571/jwks.json with env {}
15:05:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.050454 4180 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.051402 4180 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:37.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.052167 4180 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:37.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.086148 4180 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.086727 4180 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.087479 4180 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.087697 4180 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.088056 4180 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.088518 4180 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44571/jwks.json
15:05:37.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.129415 4180 master_runner.cc:386] Master server non-default flags:
15:05:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.18.20:44143
15:05:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.18.62:43303,127.4.18.61:38117,127.4.18.60:42413
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.18.62:43303
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44571/jwks.json
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data/info.pb
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.18.62
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/logs
15:05:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.130823 4180 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.133482 4180 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:37.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.151692 4187 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.152410 4186 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.153717 4180 server_base.cc:1034] running on GCE node
15:05:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.153293 4189 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.285897 4180 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:38.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.289201 4180 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:38.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.290722 4180 hybrid_clock.cc:648] HybridClock initialized: now 1736780738290664 us; error 69 us; skew 500 ppm
15:05:38.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.291605 4180 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.298597 4180 webserver.cc:458] Webserver started at http://127.4.18.62:39045/ using document root <none> and password file <none>
15:05:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.299595 4180 fs_manager.cc:362] Metadata directory not provided
15:05:38.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.299805 4180 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:38.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.300249 4180 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.305368 4180 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data/instance:
15:05:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d718fbf751e4465ae4b0465a6515eac"
15:05:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:38 on dist-test-slave-hwf5"
15:05:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.306629 4180 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal/instance:
15:05:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d718fbf751e4465ae4b0465a6515eac"
15:05:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:38 on dist-test-slave-hwf5"
15:05:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.314728 4180 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
15:05:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.321169 4196 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.322496 4180 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:05:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.322906 4180 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data,/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d718fbf751e4465ae4b0465a6515eac"
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:38 on dist-test-slave-hwf5"
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.323305 4180 fs_report.cc:389] FS layout report
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal
15:05:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data/data
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:38.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.374445 4180 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.375967 4180 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.376442 4180 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.446297 4180 rpc_server.cc:307] RPC server started. Bound to: 127.4.18.62:43303
15:05:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.446399 4247 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.18.62:43303 every 8 connection(s)
15:05:38.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.449381 4180 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/data/info.pb
15:05:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.455235 4248 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.458115 4168 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7o5v58/build/tsan/bin/kudu as pid 4180
15:05:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.458662 4168 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-0/wal/instance
15:05:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.463968 4168 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7o5v58/build/tsan/bin/kudu
15:05:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7o5v58/build/tsan/bin/kudu
15:05:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal
15:05:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data
15:05:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/logs
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data/info.pb
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.18.61:38117
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.18.61
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.18.20:44143
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.18.62:43303,127.4.18.61:38117,127.4.18.60:42413
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44571/jwks.json with env {}
15:05:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.471271 4248 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } has no permanent_uuid. Determining permanent_uuid...
15:05:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.503746 4248 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } has no permanent_uuid. Determining permanent_uuid...
15:05:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.506806 4199 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.511445 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.561120 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 1
15:05:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.565356 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.633898 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 2
15:05:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.638235 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.729761 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 3
15:05:38.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.734643 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:38.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.779403 4250 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.779982 4250 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.780544 4250 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.811961 4250 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.812295 4250 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.812487 4250 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.812669 4250 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.812876 4250 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.813022 4250 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44571/jwks.json
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.849627 4250 master_runner.cc:386] Master server non-default flags:
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.18.20:44143
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.18.62:43303,127.4.18.61:38117,127.4.18.60:42413
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.18.61:38117
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44571/jwks.json
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data/info.pb
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.18.61
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/logs
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.851083 4250 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.852835 4250 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:38.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.867467 4259 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:38.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.878094 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 4
15:05:38.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.882746 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.182219 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 5
15:05:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.185986 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:39.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.733626 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 6
15:05:39.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:39.747967 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.61:38117: Network error: Client connection negotiation failed: client connection to 127.4.18.61:38117: connect: Connection refused (error 111)
15:05:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.867491 4260 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.869638 4250 server_base.cc:1034] running on GCE node
15:05:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.868889 4262 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.974731 4250 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:39.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.978291 4250 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:39.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.979710 4250 hybrid_clock.cc:648] HybridClock initialized: now 1736780739979667 us; error 49 us; skew 500 ppm
15:05:39.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.980746 4250 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:39.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.993983 4250 webserver.cc:458] Webserver started at http://127.4.18.61:38007/ using document root <none> and password file <none>
15:05:39.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.995275 4250 fs_manager.cc:362] Metadata directory not provided
15:05:39.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.995527 4250 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.996140 4250 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:40.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.001317 4250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data/instance:
15:05:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293a44559b84410db909c064dc8ebd7b"
15:05:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-hwf5"
15:05:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.002708 4250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal/instance:
15:05:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293a44559b84410db909c064dc8ebd7b"
15:05:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-hwf5"
15:05:40.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.010417 4250 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
15:05:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.016379 4271 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.017630 4250 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
15:05:40.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.018047 4250 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data,/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal
15:05:40.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293a44559b84410db909c064dc8ebd7b"
15:05:40.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-hwf5"
15:05:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.018460 4250 fs_report.cc:389] FS layout report
15:05:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal
15:05:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal
15:05:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data/data
15:05:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:40.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.072453 4250 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:40.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.074105 4250 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:40.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.074630 4250 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:40.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.144748 4250 rpc_server.cc:307] RPC server started. Bound to: 127.4.18.61:38117
15:05:40.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.144840 4322 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.18.61:38117 every 8 connection(s)
15:05:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.147614 4250 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/data/info.pb
15:05:40.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.153076 4323 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.154948 4168 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7o5v58/build/tsan/bin/kudu as pid 4250
15:05:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.155337 4168 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-1/wal/instance
15:05:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.157486 4168 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7o5v58/build/tsan/bin/kudu
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7o5v58/build/tsan/bin/kudu
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/logs
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data/info.pb
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.18.60:42413
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.18.60
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.18.20:44143
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.18.62:43303,127.4.18.61:38117,127.4.18.60:42413
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44571/jwks.json with env {}
15:05:40.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.166648 4323 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } has no permanent_uuid. Determining permanent_uuid...
15:05:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.197487 4323 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } has no permanent_uuid. Determining permanent_uuid...
15:05:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.214006 4323 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } has no permanent_uuid. Determining permanent_uuid...
15:05:40.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.216708 4274 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.221465 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.271049 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 1
15:05:40.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.275269 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.343668 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 2
15:05:40.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.348325 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.439781 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 3
15:05:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.444686 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.490674 4325 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:40.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.491366 4325 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.492039 4325 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.523365 4325 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.523716 4325 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.524039 4325 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.524399 4325 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.524677 4325 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.524951 4325 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44571/jwks.json
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.560489 4325 master_runner.cc:386] Master server non-default flags:
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.18.20:44143
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.18.62:43303,127.4.18.61:38117,127.4.18.60:42413
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.18.60:42413
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44571/jwks.json
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data/info.pb
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.18.60
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/logs
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.562294 4325 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.564574 4325 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.579419 4336 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.588228 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 4
15:05:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.592736 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.808562 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } attempt: 7
15:05:40.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.820231 4248 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } has no permanent_uuid. Determining permanent_uuid...
15:05:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.826011 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.884398 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 1
15:05:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.888043 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.892156 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 5
15:05:40.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.895643 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.969462 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 2
15:05:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.973599 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:41.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.058993 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 3
15:05:41.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.063184 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.203600 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 4
15:05:41.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.208108 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.443172 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 6
15:05:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.454295 4323 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:41.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.491536 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 5
15:05:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.495950 4248 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.18.60:42413: Network error: Client connection negotiation failed: client connection to 127.4.18.60:42413: connect: Connection refused (error 111)
15:05:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.580061 4335 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.581887 4325 server_base.cc:1034] running on GCE node
15:05:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.582598 4338 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.709461 4325 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:41.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.712385 4325 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.713757 4325 hybrid_clock.cc:648] HybridClock initialized: now 1736780741713718 us; error 47 us; skew 500 ppm
15:05:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.714584 4325 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.721052 4325 webserver.cc:458] Webserver started at http://127.4.18.60:46197/ using document root <none> and password file <none>
15:05:41.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.722016 4325 fs_manager.cc:362] Metadata directory not provided
15:05:41.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.722265 4325 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.722764 4325 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.727659 4325 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data/instance:
15:05:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58317c4e165c4daa90d480ceb471babf"
15:05:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-hwf5"
15:05:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.729013 4325 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal/instance:
15:05:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58317c4e165c4daa90d480ceb471babf"
15:05:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-hwf5"
15:05:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.736128 4325 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
15:05:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.741904 4350 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.743041 4325 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:05:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.743391 4325 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data,/tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal
15:05:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58317c4e165c4daa90d480ceb471babf"
15:05:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-hwf5"
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.743738 4325 fs_report.cc:389] FS layout report
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data/data
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.791074 4325 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.792783 4325 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.793323 4325 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.874531 4325 rpc_server.cc:307] RPC server started. Bound to: 127.4.18.60:42413
15:05:41.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.874636 4401 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.18.60:42413 every 8 connection(s)
15:05:41.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.877388 4325 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/data/info.pb
15:05:41.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.883262 4402 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:41.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.883445 4168 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7o5v58/build/tsan/bin/kudu as pid 4325
15:05:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.883888 4168 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7o5v58/test-tmp/mini-kudu-cluster710733401570456903/master-2/wal/instance
15:05:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.898818 4402 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.929802 4402 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.942910 4402 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } has no permanent_uuid. Determining permanent_uuid...
15:05:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.961545 4402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Bootstrap starting.
15:05:41.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.969173 4402 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Neither blocks nor log segments found. Creating new log.
15:05:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.971215 4402 log.cc:826] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Log is configured to *not* fsync() on all Append() calls
15:05:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.977638 4402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: No bootstrap required, opened a new log
15:05:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.005146 4402 raft_consensus.cc:357] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:42.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.005810 4402 raft_consensus.cc:383] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:42.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.005995 4402 raft_consensus.cc:738] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58317c4e165c4daa90d480ceb471babf, State: Initialized, Role: FOLLOWER
15:05:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.006736 4402 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:42.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.009742 4411 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.010764 4411 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: This master's current role is: FOLLOWER
15:05:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.011924 4402 sys_catalog.cc:564] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: configured and running, proceeding with master startup.
15:05:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.048343 4248 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 6
15:05:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.048440 4356 tablet.cc:2367] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:42.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.053859 4422 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:42.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:42.054255 4422 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.054668 4411 raft_consensus.cc:491] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.055223 4411 raft_consensus.cc:513] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.061355 4411 leader_election.cc:290] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7d718fbf751e4465ae4b0465a6515eac (127.4.18.62:43303), 293a44559b84410db909c064dc8ebd7b (127.4.18.61:38117)
15:05:42.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.043079 4325 master_runner.cc:186] Error getting master registration for 127.4.18.62:43303: OK, instance_id { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" instance_seqno: 1736780738427057 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:42.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.082962 4325 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.085973 4248 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: Bootstrap starting.
15:05:42.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.101580 4248 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: Neither blocks nor log segments found. Creating new log.
15:05:42.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.104633 4248 log.cc:826] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: Log is configured to *not* fsync() on all Append() calls
15:05:42.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.112438 4248 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: No bootstrap required, opened a new log
15:05:42.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.159787 4248 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.161175 4248 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.161486 4248 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d718fbf751e4465ae4b0465a6515eac, State: Initialized, Role: FOLLOWER
15:05:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.162438 4248 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:42.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.167977 4248 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: configured and running, proceeding with master startup.
15:05:42.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.168319 4223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "58317c4e165c4daa90d480ceb471babf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d718fbf751e4465ae4b0465a6515eac" is_pre_election: true
15:05:42.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.169281 4223 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 58317c4e165c4daa90d480ceb471babf in term 0.
15:05:42.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.166303 4426 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.171555 4352 leader_election.cc:304] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 58317c4e165c4daa90d480ceb471babf, 7d718fbf751e4465ae4b0465a6515eac; no voters:
15:05:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.171975 4426 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: This master's current role is: FOLLOWER
15:05:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.172672 4411 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Leader pre-election won for term 1
15:05:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.173070 4411 raft_consensus.cc:491] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:05:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.173393 4411 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 0 FOLLOWER]: Advancing to term 1
15:05:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.602941 4323 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } attempt: 7
15:05:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.616775 4202 tablet.cc:2367] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.635073 4431 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.643469 4431 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:44.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.648435 4411 consensus_meta.cc:287] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Time spent flushing consensus metadata: real 2.475s user 0.010s sys 0.000s
15:05:44.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.648813 4411 raft_consensus.cc:513] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.649770 4377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d718fbf751e4465ae4b0465a6515eac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58317c4e165c4daa90d480ceb471babf" is_pre_election: true
15:05:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.650874 4377 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 7d718fbf751e4465ae4b0465a6515eac in current term 1: Already voted for candidate 58317c4e165c4daa90d480ceb471babf in this term.
15:05:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.653466 4431 leader_election.cc:290] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293a44559b84410db909c064dc8ebd7b (127.4.18.61:38117), 58317c4e165c4daa90d480ceb471babf (127.4.18.60:42413)
15:05:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.661393 4422 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.661741 4422 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.666433 4223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "58317c4e165c4daa90d480ceb471babf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d718fbf751e4465ae4b0465a6515eac"
15:05:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.667210 4223 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 0 FOLLOWER]: Advancing to term 1
15:05:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.677891 4323 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: Bootstrap starting.
15:05:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.680773 4223 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 58317c4e165c4daa90d480ceb471babf in term 1.
15:05:44.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.692946 4411 leader_election.cc:290] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [CANDIDATE]: Term 1 election: Requested vote from peers 7d718fbf751e4465ae4b0465a6515eac (127.4.18.62:43303), 293a44559b84410db909c064dc8ebd7b (127.4.18.61:38117)
15:05:44.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.696300 4352 leader_election.cc:304] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 58317c4e165c4daa90d480ceb471babf, 7d718fbf751e4465ae4b0465a6515eac; no voters:
15:05:44.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.710299 4411 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 1 FOLLOWER]: Leader election won for term 1
15:05:44.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.714022 4323 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: Neither blocks nor log segments found. Creating new log.
15:05:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.715327 4411 raft_consensus.cc:695] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [term 1 LEADER]: Becoming Leader. State: Replica: 58317c4e165c4daa90d480ceb471babf, State: Running, Role: LEADER
15:05:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.720170 4323 log.cc:826] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: Log is configured to *not* fsync() on all Append() calls
15:05:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.736215 4323 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: No bootstrap required, opened a new log
15:05:44.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.725613 4411 consensus_queue.cc:237] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:44.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.756080 4248 thread.cc:635] rpc reactor (reactor) Time spent creating pthread: real 2.567s user 1.762s sys 0.732s
15:05:44.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.756561 4248 thread.cc:601] rpc reactor (reactor) Time spent starting thread: real 2.568s user 1.766s sys 0.732s
15:05:44.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.789853 4323 raft_consensus.cc:357] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.791239 4323 raft_consensus.cc:383] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.791806 4323 raft_consensus.cc:738] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 293a44559b84410db909c064dc8ebd7b, State: Initialized, Role: FOLLOWER
15:05:44.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.793239 4323 consensus_queue.cc:260] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.767858 4248 thread.cc:635] rpc reactor (reactor) Time spent creating pthread: real 0.982s user 0.000s sys 0.041s
15:05:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.768318 4248 thread.cc:601] rpc reactor (reactor) Time spent starting thread: real 0.983s user 0.000s sys 0.042s
15:05:45.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.790731 4438 sys_catalog.cc:455] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.792079 4438 sys_catalog.cc:458] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: This master's current role is: FOLLOWER
15:05:45.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.791692 4437 raft_consensus.cc:491] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:45.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.798241 4437 raft_consensus.cc:513] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } }
15:05:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.804266 4437 leader_election.cc:290] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7d718fbf751e4465ae4b0465a6515eac (127.4.18.62:43303), 58317c4e165c4daa90d480ceb471babf (127.4.18.60:42413)
15:05:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.805212 4223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "293a44559b84410db909c064dc8ebd7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d718fbf751e4465ae4b0465a6515eac" is_pre_election: true
15:05:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.806102 4223 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 293a44559b84410db909c064dc8ebd7b in current term 1: Already voted for candidate 58317c4e165c4daa90d480ceb471babf in this term.
15:05:45.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.796780 4411 thread.cc:635] wal-append [worker] (thread pool) Time spent creating pthread: real 0.988s user 0.001s sys 0.000s
15:05:45.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.810038 4353 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.4.18.61:38117 timed out after 2.208s (SENT)
15:05:45.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.812512 4411 thread.cc:601] wal-append [worker] (thread pool) Time spent starting thread: real 1.029s user 0.001s sys 0.000s
15:05:45.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.816215 4323 thread.cc:635] raft [worker] (thread pool) Time spent creating pthread: real 1.022s user 0.798s sys 0.173s
15:05:45.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.816545 4323 thread.cc:601] raft [worker] (thread pool) Time spent starting thread: real 1.022s user 0.798s sys 0.173s
15:05:45.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.828362 4323 sys_catalog.cc:564] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: configured and running, proceeding with master startup.
15:05:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.828742 4298 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "58317c4e165c4daa90d480ceb471babf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "293a44559b84410db909c064dc8ebd7b" is_pre_election: true
15:05:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.829574 4298 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 58317c4e165c4daa90d480ceb471babf in term 0.
15:05:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.830068 4298 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:58360 (request call id 1) took 3769 ms (client timeout 2208 ms). Trace:
15:05:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.830390 4298 rpcz_store.cc:269] 0113 15:05:42.060410 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:42.060589 (+ 179us) service_pool.cc:225] Handling call
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:45.829997 (+3769408us) inbound_call.cc:173] Queueing success response
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.831537 4296 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "58317c4e165c4daa90d480ceb471babf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "293a44559b84410db909c064dc8ebd7b"
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.832489 4296 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 0 FOLLOWER]: Advancing to term 1
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.832386 4297 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d718fbf751e4465ae4b0465a6515eac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "293a44559b84410db909c064dc8ebd7b" is_pre_election: true
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.833488 4297 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:58338 (request call id 1) took 1186 ms. Trace:
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.833784 4297 rpcz_store.cc:276] 0113 15:05:44.646996 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:44.647128 (+ 132us) service_pool.cc:225] Handling call
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:45.833427 (+1186299us) inbound_call.cc:173] Queueing success response
15:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:05:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.835234 4199 leader_election.cc:304] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7d718fbf751e4465ae4b0465a6515eac; no voters: 293a44559b84410db909c064dc8ebd7b, 58317c4e165c4daa90d480ceb471babf
15:05:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.828246 4377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "293a44559b84410db909c064dc8ebd7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58317c4e165c4daa90d480ceb471babf" is_pre_election: true
15:05:45.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.846547 4274 leader_election.cc:304] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 293a44559b84410db909c064dc8ebd7b; no voters: 58317c4e165c4daa90d480ceb471babf, 7d718fbf751e4465ae4b0465a6515eac
15:05:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.848119 4298 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "58317c4e165c4daa90d480ceb471babf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "293a44559b84410db909c064dc8ebd7b" is_pre_election: true
15:05:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.845265 4422 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.851063 4296 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 58317c4e165c4daa90d480ceb471babf in term 1.
15:05:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.851537 4437 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.859155 4367 rpcz_store.cc:275] Call kudu.master.MasterService.ConnectToMaster from 127.0.0.1:51866 (request call id 17) took 1104 ms. Trace:
15:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.859526 4367 rpcz_store.cc:276] 0113 15:05:44.737856 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:44.738096 (+ 240us) service_pool.cc:225] Handling call
15:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:45.839038 (+1100942us) spinlock_profiling.cc:243] Waited 1.08 s on lock 0x7b54000316d4. stack: 00007f94d4342853 00007f94d4342750 00007f94d3eb7584 00000000004fcd24 00000000004fccb9 00007f94db242d50 00007f94db2c5f2a 00007f94dcf6adec 00007f94dd066009 00007f94dadc3544 00007f94dadc34d2 00007f94dadc3401 00007f94dadc337c 00007f94dadc2632 00007f94d908adfc 00007f94d908a236
15:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:45.842543 (+ 3505us) inbound_call.cc:173] Queueing success response
15:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":2488834432}
15:05:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.850425 4422 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.861313 4296 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:58360 (request call id 2) took 1169 ms. Trace:
15:05:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.860419 4411 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [LEADER]: Connected to new peer: Peer: permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
15:05:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.861654 4296 rpcz_store.cc:276] 0113 15:05:44.692375 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:05:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:44.692545 (+ 170us) service_pool.cc:225] Handling call
15:05:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:05:45.861248 (+1168703us) inbound_call.cc:173] Queueing success response
15:05:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:05:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.863543 4432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 58317c4e165c4daa90d480ceb471babf. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.864326 4432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: This master's current role is: LEADER
15:05:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.870622 4445 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:05:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.880952 4432 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [LEADER]: Connected to new peer: Peer: permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
15:05:45.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.887665 4449 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:05:45.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.910804 4453 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:05:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.917640 4437 sys_catalog.cc:455] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 58317c4e165c4daa90d480ceb471babf. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.918421 4437 sys_catalog.cc:458] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: This master's current role is: FOLLOWER
15:05:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.910526 4445 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: SysCatalogTable state changed. Reason: New leader 58317c4e165c4daa90d480ceb471babf. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.929558 4445 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: This master's current role is: FOLLOWER
15:05:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.930814 4449 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:05:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.941946 4432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.942718 4432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: This master's current role is: LEADER
15:05:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.943962 4432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.944732 4432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf [sys.catalog]: This master's current role is: LEADER
15:05:45.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.948330 4445 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.949098 4445 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac [sys.catalog]: This master's current role is: FOLLOWER
15:05:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.964305 4437 sys_catalog.cc:455] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "58317c4e165c4daa90d480ceb471babf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7d718fbf751e4465ae4b0465a6515eac" member_type: VOTER last_known_addr { host: "127.4.18.62" port: 43303 } } peers { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" member_type: VOTER last_known_addr { host: "127.4.18.61" port: 38117 } } peers { permanent_uuid: "58317c4e165c4daa90d480ceb471babf" member_type: VOTER last_known_addr { host: "127.4.18.60" port: 42413 } } }
15:05:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.965085 4437 sys_catalog.cc:458] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b [sys.catalog]: This master's current role is: FOLLOWER
15:05:46.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.920272 4180 master_runner.cc:186] Error getting master registration for 127.4.18.61:38117: OK, instance_id { permanent_uuid: "293a44559b84410db909c064dc8ebd7b" instance_seqno: 1736780740123770 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.004539 4180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:46.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.057610 4449 catalog_manager.cc:1348] Generated new cluster ID: f8ad3701c7574a5186b8540c4f8902a1
15:05:46.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.057950 4449 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:05:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:46.077701 4467 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:46.078500 4467 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.096838 4449 catalog_manager.cc:1371] Generated new certificate authority record
15:05:46.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.116303 4449 catalog_manager.cc:1505] Loading token signing keys...
15:05:47.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) F20250113 15:05:47.070068 4168 external_mini_cluster.cc:638] Check failed: leader_master() != nullptr Must have started at least 1 master before adding tablet servers
15:05:47.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.088865 4467 catalog_manager.cc:1260] Loaded cluster ID: f8ad3701c7574a5186b8540c4f8902a1
15:05:47.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.089179 4467 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: loading cluster ID for follower catalog manager: success
15:05:47.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Check failure stack trace: ***
15:05:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.117838 4467 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 293a44559b84410db909c064dc8ebd7b: acquiring CA information for follower catalog manager: success
15:05:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.134712 4467 catalog_manager.cc:874] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
15:05:47.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.169742 4453 catalog_manager.cc:1260] Loaded cluster ID: f8ad3701c7574a5186b8540c4f8902a1
15:05:47.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.170460 4453 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: loading cluster ID for follower catalog manager: success
15:05:47.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f152c313950 google::LogMessage::Flush()
15:05:47.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.226835 4449 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Generated new TSK 0
15:05:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.228559 4449 catalog_manager.cc:1506] Time spent T 00000000000000000000000000000000 P 58317c4e165c4daa90d480ceb471babf: Loading token signing keys: real 1.111s user 0.819s sys 0.143s
15:05:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.228878 4449 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:05:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.262533 4453 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 7d718fbf751e4465ae4b0465a6515eac: acquiring CA information for follower catalog manager: success
15:05:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f152c318a8b google::LogMessageFatal::~LogMessageFatal()
15:05:47.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f15376f5c1d kudu::cluster::ExternalMiniCluster::AddTabletServer()
15:05:47.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.303488 4453 catalog_manager.cc:874] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
15:05:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f15376f3c9b kudu::cluster::ExternalMiniCluster::Start()
15:05:47.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6510aa kudu::tools::(anonymous namespace)::ProcessRequest()
15:05:47.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64f9a5 kudu::tools::(anonymous namespace)::RunControlShell()
15:05:47.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x529a02 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
15:05:47.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x52996d std::__1::__invoke_void_return_wrapper<>::__call<>()
15:05:47.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x5298dd std::__1::__function::__alloc_func<>::operator()()
15:05:47.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x528546 std::__1::__function::__func<>::operator()()
15:05:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1537466f90 std::__1::__function::__value_func<>::operator()()
15:05:47.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f15374597f2 std::__1::function<>::operator()()
15:05:47.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f1537457583 kudu::tools::Action::Run()
15:05:47.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6455a1 kudu::tools::DispatchCommand()
15:05:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x645ed8 kudu::tools::RunTool()
15:05:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64641f main
15:05:47.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f152aeb8bf7 __libc_start_main
15:05:47.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x43399a _start
15:05:48.655 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduPartitioner.testBuildTimeout: failed attempt 1
java.io.EOFException: null
at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_275]
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)