02:06:19.577 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:06:19.591 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0Rct8h/test-tmp
02:06:19.594 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:06:19.597 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0Rct8h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:06:19.925 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439"
extra_master_flags: "--rpc_negotiation_inject_delay_ms=1000"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:06:19.994 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:06:19.995 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:06:19.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:19.998535 31737 mini_oidc.cc:150] Starting JWKS server
02:06:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.002321 31737 webserver.cc:469] Webserver started at http://127.0.0.1:33035/ using document root <none> and password file <none>
02:06:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.002785 31737 mini_oidc.cc:186] Starting OIDC Discovery server
02:06:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.004455 31737 webserver.cc:469] Webserver started at http://127.0.0.1:33719/ using document root <none> and password file <none>
02:06:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T02:06:20Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:06:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T02:06:20Z Disabled control of system clock
02:06:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.067426 31737 external_mini_cluster.cc:1351] Running /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/logs
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data/info.pb
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.126:37411
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.126
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:35457
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:37411,127.30.254.125:38191,127.30.254.124:40933
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33035/jwks.json
02:06:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_negotiation_inject_delay_ms=1000 with env {}
02:06:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.369580 31749 flags.cc:425] Enabled unsafe flag: --rpc_negotiation_inject_delay_ms=1000
02:06:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.370086 31749 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.370383 31749 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.370955 31749 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:20.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.401933 31749 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:20.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.402261 31749 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:20.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.402480 31749 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.402796 31749 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.403079 31749 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.403326 31749 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33035/jwks.json
02:06:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.456506 31749 master_runner.cc:386] Master server non-default flags:
02:06:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:35457
02:06:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:37411,127.30.254.125:38191,127.30.254.124:40933
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_negotiation_inject_delay_ms=1000
02:06:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.126:37411
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33035/jwks.json
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data/info.pb
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.126
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/logs
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:06:20.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6789
02:06:20.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:20.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.457904 31749 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:20.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:20.459647 31749 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:20.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.475519 31756 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.478016 31758 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:20.475931 31755 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:21.670266 31757 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.670610 31749 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.674608 31749 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.677613 31749 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:21.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.679023 31749 hybrid_clock.cc:648] HybridClock initialized: now 1750989981678971 us; error 56 us; skew 500 ppm
02:06:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.679999 31749 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.686821 31749 webserver.cc:469] Webserver started at http://127.30.254.126:35983/ using document root <none> and password file <none>
02:06:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.687908 31749 fs_manager.cc:362] Metadata directory not provided
02:06:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.688155 31749 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.688766 31749 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.693704 31749 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data/instance:
02:06:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0634f06f5a445b08e76834f32aa083f"
02:06:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:21 on dist-test-slave-3tm1"
02:06:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.695101 31749 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal/instance:
02:06:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0634f06f5a445b08e76834f32aa083f"
02:06:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:21 on dist-test-slave-3tm1"
02:06:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.702690 31749 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
02:06:21.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.708632 31765 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:21.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.709720 31749 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
02:06:21.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.710078 31749 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data,/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal
02:06:21.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0634f06f5a445b08e76834f32aa083f"
02:06:21.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:21 on dist-test-slave-3tm1"
02:06:21.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.710482 31749 fs_report.cc:389] FS layout report
02:06:21.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:21.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal
02:06:21.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal
02:06:21.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data/data
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:21.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.761613 31749 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:21.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.763245 31749 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:21.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.763743 31749 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.843173 31749 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.126:37411
02:06:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.843245 31816 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.254.126:37411 every 8 connection(s)
02:06:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.846048 31749 server_base.cc:1180] Dumped server information to /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/data/info.pb
02:06:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.851914 31817 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.854609 31737 external_mini_cluster.cc:1413] Started /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu as pid 31749
02:06:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.855118 31737 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-0/wal/instance
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.860947 31737 external_mini_cluster.cc:1351] Running /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/logs
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data/info.pb
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.125:38191
02:06:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.125
02:06:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:35457
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:37411,127.30.254.125:38191,127.30.254.124:40933
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33035/jwks.json
02:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_negotiation_inject_delay_ms=1000 with env {}
02:06:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:21.868505 31817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } has no permanent_uuid. Determining permanent_uuid...
02:06:21.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:21.877840 31821 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.166669 31819 flags.cc:425] Enabled unsafe flag: --rpc_negotiation_inject_delay_ms=1000
02:06:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.167146 31819 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.167456 31819 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.168186 31819 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:22.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.199945 31819 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:22.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.200270 31819 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:22.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.200469 31819 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:22.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.200721 31819 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.201018 31819 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.201284 31819 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33035/jwks.json
02:06:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:22.237358 31819 master_runner.cc:386] Master server non-default flags:
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:35457
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:37411,127.30.254.125:38191,127.30.254.124:40933
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_negotiation_inject_delay_ms=1000
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.125:38191
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33035/jwks.json
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data/info.pb
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.125
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/logs
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6789
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:22.238741 31819 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:22.240689 31819 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.259384 31828 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:22.892269 31817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } has no permanent_uuid. Determining permanent_uuid...
02:06:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.895364 31768 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.900312 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.125:38191: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:22.950131 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } attempt: 1
02:06:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.954406 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.125:38191: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.023015 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } attempt: 2
02:06:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.028436 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.125:38191: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.119849 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } attempt: 3
02:06:23.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.124702 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.125:38191: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:23.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.268132 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } attempt: 4
02:06:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.272549 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.125:38191: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.262317 31827 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:22.263233 31819 server_base.cc:1048] running on GCE node
02:06:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:22.263139 31830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:23.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.471580 31819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.474865 31819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.476254 31819 hybrid_clock.cc:648] HybridClock initialized: now 1750989983476182 us; error 90 us; skew 500 ppm
02:06:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.477186 31819 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.484508 31819 webserver.cc:469] Webserver started at http://127.30.254.125:46479/ using document root <none> and password file <none>
02:06:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.485503 31819 fs_manager.cc:362] Metadata directory not provided
02:06:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.485735 31819 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.486285 31819 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.490835 31819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data/instance:
02:06:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ebb916052bc64ed597293b7f8e9d6da9"
02:06:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:23 on dist-test-slave-3tm1"
02:06:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.492168 31819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal/instance:
02:06:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ebb916052bc64ed597293b7f8e9d6da9"
02:06:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:23 on dist-test-slave-3tm1"
02:06:23.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.499841 31819 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
02:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.505999 31838 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.507112 31819 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.507483 31819 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data,/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal
02:06:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ebb916052bc64ed597293b7f8e9d6da9"
02:06:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:23 on dist-test-slave-3tm1"
02:06:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.507872 31819 fs_report.cc:389] FS layout report
02:06:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data/data
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.569844 31819 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.571488 31819 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.572026 31819 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.571971 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } attempt: 5
02:06:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.576570 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.125:38191: Network error: Client connection negotiation failed: client connection to 127.30.254.125:38191: connect: Connection refused (error 111)
02:06:23.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.654363 31819 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.125:38191
02:06:23.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.655015 31889 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.254.125:38191 every 8 connection(s)
02:06:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.657039 31819 server_base.cc:1180] Dumped server information to /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/data/info.pb
02:06:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.661046 31737 external_mini_cluster.cc:1413] Started /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu as pid 31819
02:06:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.661517 31737 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-1/wal/instance
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.663717 31890 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.663659 31737 external_mini_cluster.cc:1351] Running /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/logs
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data/info.pb
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.124:40933
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.124
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:35457
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:37411,127.30.254.125:38191,127.30.254.124:40933
02:06:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33035/jwks.json
02:06:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_negotiation_inject_delay_ms=1000 with env {}
02:06:23.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:23.676604 31890 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } has no permanent_uuid. Determining permanent_uuid...
02:06:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.682703 31893 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:23.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.970820 31892 flags.cc:425] Enabled unsafe flag: --rpc_negotiation_inject_delay_ms=1000
02:06:23.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.971310 31892 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:23.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.971578 31892 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:23.972034 31892 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:24.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.003137 31892 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:24.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.003465 31892 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:24.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.003659 31892 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.003895 31892 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.004197 31892 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.004731 31892 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33035/jwks.json
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:24.043730 31892 master_runner.cc:386] Master server non-default flags:
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:35457
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:37411,127.30.254.125:38191,127.30.254.124:40933
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_negotiation_inject_delay_ms=1000
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.124:40933
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33035/jwks.json
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data/info.pb
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.124
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/logs
02:06:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6789
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:24.045270 31892 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:24.046995 31892 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.063200 31901 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:24.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:24.123994 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } attempt: 6
02:06:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.142688 31907 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:24.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:24.702056 31890 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } has no permanent_uuid. Determining permanent_uuid...
02:06:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.708207 31908 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.160859 31817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } has no permanent_uuid. Determining permanent_uuid...
02:06:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.167964 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.124:40933: Network error: Client connection negotiation failed: client connection to 127.30.254.124:40933: connect: Connection refused (error 111)
02:06:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.220368 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } attempt: 1
02:06:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.226104 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.124:40933: Network error: Client connection negotiation failed: client connection to 127.30.254.124:40933: connect: Connection refused (error 111)
02:06:25.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.064036 31900 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:24.065845 31903 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.300944 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } attempt: 2
02:06:25.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.305330 31902 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1241 milliseconds
02:06:25.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.305465 31892 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:25.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.306070 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.124:40933: Network error: Client connection negotiation failed: client connection to 127.30.254.124:40933: connect: Connection refused (error 111)
02:06:25.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.307245 31892 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.309829 31892 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.311195 31892 hybrid_clock.cc:648] HybridClock initialized: now 1750989985311149 us; error 61 us; skew 500 ppm
02:06:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.312083 31892 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.318701 31892 webserver.cc:469] Webserver started at http://127.30.254.124:42153/ using document root <none> and password file <none>
02:06:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.319643 31892 fs_manager.cc:362] Metadata directory not provided
02:06:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.319866 31892 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.320410 31892 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.324853 31892 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data/instance:
02:06:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:25 on dist-test-slave-3tm1"
02:06:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.326068 31892 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal/instance:
02:06:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:25 on dist-test-slave-3tm1"
02:06:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.334932 31892 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
02:06:25.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.341061 31914 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.342098 31892 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:06:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.342430 31892 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data,/tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal
02:06:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 02:06:25 on dist-test-slave-3tm1"
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.342839 31892 fs_report.cc:389] FS layout report
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data/data
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.419492 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } attempt: 3
02:06:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.420003 31892 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.422410 31892 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.422969 31892 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.423681 31817 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.254.124:40933: Network error: Client connection negotiation failed: client connection to 127.30.254.124:40933: connect: Connection refused (error 111)
02:06:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.504276 31892 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.124:40933
02:06:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.504405 31965 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.254.124:40933 every 8 connection(s)
02:06:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.506871 31892 server_base.cc:1180] Dumped server information to /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/data/info.pb
02:06:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.510891 31737 external_mini_cluster.cc:1413] Started /tmp/dist-test-task0Rct8h/build/tsan/bin/kudu as pid 31892
02:06:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.511363 31737 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task0Rct8h/test-tmp/mini-kudu-cluster1328265153638321439/master-2/wal/instance
02:06:25.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.513366 31966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.518116 31968 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:25.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.521138 31907 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.524753 31970 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:25.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.527925 31966 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } has no permanent_uuid. Determining permanent_uuid...
02:06:25.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.534293 31971 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.573160 31817 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } attempt: 4
02:06:25.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.576835 31973 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:25.720393 31890 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } has no permanent_uuid. Determining permanent_uuid...
02:06:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:25.725433 31974 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.546993 31966 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } has no permanent_uuid. Determining permanent_uuid...
02:06:26.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.549718 31907 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.592792 31817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: Bootstrap starting.
02:06:26.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.598466 31817 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: Neither blocks nor log segments found. Creating new log.
02:06:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.600947 31817 log.cc:826] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: Log is configured to *not* fsync() on all Append() calls
02:06:26.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.606029 31817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: No bootstrap required, opened a new log
02:06:26.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.624991 31817 raft_consensus.cc:357] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.626274 31817 raft_consensus.cc:383] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:26.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.626932 31817 raft_consensus.cc:738] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c0634f06f5a445b08e76834f32aa083f, State: Initialized, Role: FOLLOWER
02:06:26.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.627914 31817 consensus_queue.cc:260] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.631011 31976 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.631670 31976 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: This master's current role is: FOLLOWER
02:06:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.632582 31817 sys_catalog.cc:564] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: configured and running, proceeding with master startup.
02:06:26.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.667443 31987 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:26.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.668188 31987 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:26.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.659695 31749 master_runner.cc:186] Error getting master registration for 127.30.254.125:38191: OK, instance_id { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" instance_seqno: 1750989983632615 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:26.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.691145 31749 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.713172 31976 raft_consensus.cc:491] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.713775 31976 raft_consensus.cc:513] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.716928 31976 leader_election.cc:290] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ebb916052bc64ed597293b7f8e9d6da9 (127.30.254.125:38191), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.746865 31890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: Bootstrap starting.
02:06:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.755244 31890 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: Neither blocks nor log segments found. Creating new log.
02:06:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.757562 31890 log.cc:826] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: Log is configured to *not* fsync() on all Append() calls
02:06:26.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.763271 31890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: No bootstrap required, opened a new log
02:06:26.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.774861 31771 tablet.cc:2378] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:26.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.792618 31890 raft_consensus.cc:357] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.793488 31890 raft_consensus.cc:383] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:26.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.793797 31890 raft_consensus.cc:738] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ebb916052bc64ed597293b7f8e9d6da9, State: Initialized, Role: FOLLOWER
02:06:26.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.796808 31890 consensus_queue.cc:260] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.797770 31990 raft_consensus.cc:491] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:26.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.798925 31990 raft_consensus.cc:513] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.803323 31890 sys_catalog.cc:564] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: configured and running, proceeding with master startup.
02:06:26.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.803362 31792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0634f06f5a445b08e76834f32aa083f" is_pre_election: true
02:06:26.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.803344 31990 leader_election.cc:290] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c0634f06f5a445b08e76834f32aa083f (127.30.254.126:37411), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:26.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.804697 31792 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ebb916052bc64ed597293b7f8e9d6da9 in term 0.
02:06:26.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.805727 31865 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ebb916052bc64ed597293b7f8e9d6da9" is_pre_election: true
02:06:26.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.806221 31841 leader_election.cc:304] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c0634f06f5a445b08e76834f32aa083f, ebb916052bc64ed597293b7f8e9d6da9; no voters:
02:06:26.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.806519 31865 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c0634f06f5a445b08e76834f32aa083f in term 0.
02:06:26.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.807756 31768 leader_election.cc:304] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c0634f06f5a445b08e76834f32aa083f, ebb916052bc64ed597293b7f8e9d6da9; no voters:
02:06:26.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.801633 31991 sys_catalog.cc:455] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.808723 31976 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.808938 31991 sys_catalog.cc:458] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: This master's current role is: FOLLOWER
02:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.809053 31976 raft_consensus.cc:491] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.809372 31976 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 0 FOLLOWER]: Advancing to term 1
02:06:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.823583 31976 raft_consensus.cc:513] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.832589 31844 tablet.cc:2378] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:26.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.837035 31990 raft_consensus.cc:2802] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.837221 31865 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ebb916052bc64ed597293b7f8e9d6da9"
02:06:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.837476 31990 raft_consensus.cc:491] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.837808 31990 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 0 FOLLOWER]: Advancing to term 1
02:06:26.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.842355 31976 leader_election.cc:290] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [CANDIDATE]: Term 1 election: Requested vote from peers ebb916052bc64ed597293b7f8e9d6da9 (127.30.254.125:38191), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:26.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.853020 31990 raft_consensus.cc:513] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.857743 31792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0634f06f5a445b08e76834f32aa083f"
02:06:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.858045 31990 leader_election.cc:290] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 1 election: Requested vote from peers c0634f06f5a445b08e76834f32aa083f (127.30.254.126:37411), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.858707 31792 raft_consensus.cc:2391] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate ebb916052bc64ed597293b7f8e9d6da9 in current term 1: Already voted for candidate c0634f06f5a445b08e76834f32aa083f in this term.
02:06:26.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.861637 31865 raft_consensus.cc:2391] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate c0634f06f5a445b08e76834f32aa083f in current term 1: Already voted for candidate ebb916052bc64ed597293b7f8e9d6da9 in this term.
02:06:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.982213 32002 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:26.983314 32002 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.952536 31819 master_runner.cc:186] Error getting master registration for 127.30.254.124:40933: OK, instance_id { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" instance_seqno: 1750989985482278 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:26.985184 31819 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:27.561422 31966 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } has no permanent_uuid. Determining permanent_uuid...
02:06:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:27.577486 32004 negotiation.cc:266] Injecting 1000ms delay in negotiation
02:06:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:27.672736 31987 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:27.673105 31987 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:27.988386 32002 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:27.988761 32002 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.388842 31840 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.30.254.124:40933 timed out after 1.585s (SENT)
02:06:28.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.584977 31767 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.30.254.124:40933 timed out after 1.865s (SENT) [suppressed 9 similar messages]
02:06:28.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.599109 31966 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: Bootstrap starting.
02:06:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.611990 31966 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: Neither blocks nor log segments found. Creating new log.
02:06:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.619798 31966 log.cc:826] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: Log is configured to *not* fsync() on all Append() calls
02:06:28.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.638209 31966 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: No bootstrap required, opened a new log
02:06:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.678248 31987 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.678660 31987 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:28.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.705250 31966 raft_consensus.cc:357] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:28.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.706583 31966 raft_consensus.cc:383] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:28.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.707105 31966 raft_consensus.cc:738] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f9dc0c21267e48c3b2b960693e6fbfbf, State: Initialized, Role: FOLLOWER
02:06:28.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.708263 31966 consensus_queue.cc:260] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:28.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.717211 31966 sys_catalog.cc:564] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: configured and running, proceeding with master startup.
02:06:28.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.717808 31940 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" is_pre_election: true
02:06:28.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.718724 31935 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" is_pre_election: true
02:06:28.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.720191 31936 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:28.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.720975 31937 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" is_pre_election: true
02:06:28.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.722045 31938 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:28.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.722888 31941 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" is_pre_election: true
02:06:28.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.717257 31939 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:28.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.724270 31940 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ebb916052bc64ed597293b7f8e9d6da9 in term 0.
02:06:28.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.724867 31940 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:41516 (request call id 1) took 1922 ms (client timeout 1585 ms). Trace:
02:06:28.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.725220 31940 rpcz_store.cc:269] 0627 02:06:26.802793 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:28.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:26.802931 (+ 138us) service_pool.cc:225] Handling call
02:06:28.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:28.724796 (+1921865us) inbound_call.cc:173] Queueing success response
02:06:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.728032 31840 leader_election.cc:304] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ebb916052bc64ed597293b7f8e9d6da9; no voters: c0634f06f5a445b08e76834f32aa083f, f9dc0c21267e48c3b2b960693e6fbfbf
02:06:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.730124 31941 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:41512 (request call id 1) took 2012 ms (client timeout 1865 ms). Trace:
02:06:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.731084 31939 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:41512 (request call id 2) took 1894 ms (client timeout 2168 ms). Trace:
02:06:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.731515 31939 rpcz_store.cc:269] 0627 02:06:26.836853 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:26.837001 (+ 148us) service_pool.cc:225] Handling call
02:06:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:28.731032 (+1894031us) inbound_call.cc:173] Queueing success response
02:06:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.732714 31767 leader_election.cc:304] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c0634f06f5a445b08e76834f32aa083f; no voters: ebb916052bc64ed597293b7f8e9d6da9, f9dc0c21267e48c3b2b960693e6fbfbf
02:06:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.733857 31938 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:41516 (request call id 2) took 1876 ms (client timeout 1772 ms). Trace:
02:06:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.734232 31938 rpcz_store.cc:269] 0627 02:06:26.857862 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:26.858034 (+ 172us) service_pool.cc:225] Handling call
02:06:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:28.733795 (+1875761us) inbound_call.cc:173] Queueing success response
02:06:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.730464 31941 rpcz_store.cc:269] 0627 02:06:26.717709 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:26.717843 (+ 134us) service_pool.cc:225] Handling call
02:06:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 02:06:28.730066 (+2012223us) inbound_call.cc:173] Queueing success response
02:06:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.753096 32008 sys_catalog.cc:455] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.758638 32008 sys_catalog.cc:458] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: This master's current role is: FOLLOWER
02:06:28.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.770344 32011 raft_consensus.cc:2747] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
02:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.772881 32010 raft_consensus.cc:2747] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
02:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.774693 31840 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.30.254.124:40933, user_credentials={real_user=slave}} blocked reactor thread for 46737.7us
02:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.822475 31892 master_runner.cc:263] No leader master found from master f9dc0c21267e48c3b2b960693e6fbfbf
02:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.823091 32021 catalog_manager.cc:1560] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.823570 32021 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:28.823341 31892 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.932864 31920 tablet.cc:2378] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:28.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.993645 32002 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:28.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:28.994350 32002 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:29.682827 31987 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:29.683610 31987 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:29.827327 32021 catalog_manager.cc:1560] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:29.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:29.827621 32021 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:29.998908 32002 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:29.999257 32002 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) F20250627 02:06:30.555236 31737 external_mini_cluster.cc:638] Check failed: leader_master() != nullptr Must have started at least 1 master before adding tablet servers
02:06:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Check failure stack trace: ***
02:06:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf4f1da950 google::LogMessage::Flush()
02:06:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf4f1dfa8b google::LogMessageFatal::~LogMessageFatal()
02:06:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf5a5a4a8d kudu::cluster::ExternalMiniCluster::AddTabletServer()
02:06:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf5a5a2b0b kudu::cluster::ExternalMiniCluster::Start()
02:06:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64daca kudu::tools::(anonymous namespace)::ProcessRequest()
02:06:30.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64c3c5 kudu::tools::(anonymous namespace)::RunControlShell()
02:06:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x527002 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
02:06:30.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x526f6d std::__1::__invoke_void_return_wrapper<>::__call<>()
02:06:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x526edd std::__1::__function::__alloc_func<>::operator()()
02:06:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x525b46 std::__1::__function::__func<>::operator()()
02:06:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf5a317c20 std::__1::__function::__value_func<>::operator()()
02:06:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf5a30a482 std::__1::function<>::operator()()
02:06:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf5a308213 kudu::tools::Action::Run()
02:06:30.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x641fc1 kudu::tools::DispatchCommand()
02:06:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6428f8 kudu::tools::RunTool()
02:06:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x642e3f main
02:06:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7faf4dd7fbf7 __libc_start_main
02:06:30.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x430dfa _start
02:06:30.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.684619 32025 raft_consensus.cc:491] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:30.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.685153 32025 raft_consensus.cc:513] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:30.688728 31987 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.689303 31792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0634f06f5a445b08e76834f32aa083f" is_pre_election: true
02:06:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.690295 31792 raft_consensus.cc:2391] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate f9dc0c21267e48c3b2b960693e6fbfbf in current term 1: Already voted for candidate c0634f06f5a445b08e76834f32aa083f in this term.
02:06:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:30.689275 31987 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.692437 32025 leader_election.cc:290] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c0634f06f5a445b08e76834f32aa083f (127.30.254.126:37411), ebb916052bc64ed597293b7f8e9d6da9 (127.30.254.125:38191)
02:06:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.694360 31865 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ebb916052bc64ed597293b7f8e9d6da9" is_pre_election: true
02:06:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.695154 31865 raft_consensus.cc:2391] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate f9dc0c21267e48c3b2b960693e6fbfbf in current term 1: Already voted for candidate ebb916052bc64ed597293b7f8e9d6da9 in this term.
02:06:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.696409 31917 leader_election.cc:304] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f9dc0c21267e48c3b2b960693e6fbfbf; no voters: c0634f06f5a445b08e76834f32aa083f, ebb916052bc64ed597293b7f8e9d6da9
02:06:30.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.698038 32025 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 0 FOLLOWER]: Advancing to term 1
02:06:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.704991 32025 raft_consensus.cc:2747] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:06:30.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.714025 32026 raft_consensus.cc:491] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.714419 32026 raft_consensus.cc:513] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.716279 32026 leader_election.cc:290] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c0634f06f5a445b08e76834f32aa083f (127.30.254.126:37411), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.718071 31935 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" is_pre_election: true
02:06:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.718686 31935 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ebb916052bc64ed597293b7f8e9d6da9 in term 1.
02:06:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.718613 31792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0634f06f5a445b08e76834f32aa083f" is_pre_election: true
02:06:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.719158 31792 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ebb916052bc64ed597293b7f8e9d6da9 in term 1.
02:06:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.720255 31840 leader_election.cc:304] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ebb916052bc64ed597293b7f8e9d6da9, f9dc0c21267e48c3b2b960693e6fbfbf; no voters:
02:06:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.721855 32026 raft_consensus.cc:2802] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:06:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.722385 32026 raft_consensus.cc:491] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.722908 32026 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 1 FOLLOWER]: Advancing to term 2
02:06:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.730769 32026 raft_consensus.cc:513] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.732859 32027 raft_consensus.cc:491] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.733645 32027 raft_consensus.cc:513] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:30.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.734737 31792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0634f06f5a445b08e76834f32aa083f"
02:06:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.735813 31792 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 1 FOLLOWER]: Advancing to term 2
02:06:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.735747 31935 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ebb916052bc64ed597293b7f8e9d6da9" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf"
02:06:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.736253 31935 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 1 FOLLOWER]: Advancing to term 2
02:06:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.743220 31935 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ebb916052bc64ed597293b7f8e9d6da9 in term 2.
02:06:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.744494 31840 leader_election.cc:304] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ebb916052bc64ed597293b7f8e9d6da9, f9dc0c21267e48c3b2b960693e6fbfbf; no voters:
02:06:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.746573 32028 raft_consensus.cc:2802] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 2 FOLLOWER]: Leader election won for term 2
02:06:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.746582 32026 leader_election.cc:290] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [CANDIDATE]: Term 2 election: Requested vote from peers c0634f06f5a445b08e76834f32aa083f (127.30.254.126:37411), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.753425 31792 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ebb916052bc64ed597293b7f8e9d6da9 in term 2.
02:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.756268 31865 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ebb916052bc64ed597293b7f8e9d6da9" is_pre_election: true
02:06:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.756526 31935 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c0634f06f5a445b08e76834f32aa083f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" is_pre_election: true
02:06:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.756378 32027 leader_election.cc:290] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ebb916052bc64ed597293b7f8e9d6da9 (127.30.254.125:38191), f9dc0c21267e48c3b2b960693e6fbfbf (127.30.254.124:40933)
02:06:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.757257 31935 raft_consensus.cc:2391] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c0634f06f5a445b08e76834f32aa083f in current term 2: Already voted for candidate ebb916052bc64ed597293b7f8e9d6da9 in this term.
02:06:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.753918 32028 raft_consensus.cc:695] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [term 2 LEADER]: Becoming Leader. State: Replica: ebb916052bc64ed597293b7f8e9d6da9, State: Running, Role: LEADER
02:06:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.759660 32028 consensus_queue.cc:237] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } }
02:06:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.765951 31768 leader_election.cc:304] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c0634f06f5a445b08e76834f32aa083f; no voters: ebb916052bc64ed597293b7f8e9d6da9, f9dc0c21267e48c3b2b960693e6fbfbf
02:06:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.766804 32027 raft_consensus.cc:2747] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:06:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.767665 32026 sys_catalog.cc:455] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ebb916052bc64ed597293b7f8e9d6da9. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.768373 32026 sys_catalog.cc:458] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: This master's current role is: LEADER
02:06:30.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.770696 32031 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:30.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.777031 32031 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.792608 31792 raft_consensus.cc:1273] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [term 2 FOLLOWER]: Refusing update from remote peer ebb916052bc64ed597293b7f8e9d6da9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 2. (index mismatch)
02:06:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.792665 31935 raft_consensus.cc:1273] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [term 2 FOLLOWER]: Refusing update from remote peer ebb916052bc64ed597293b7f8e9d6da9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 2. (index mismatch)
02:06:30.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.794261 32026 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:30.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.795192 32028 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:30.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.824779 32025 sys_catalog.cc:455] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: SysCatalogTable state changed. Reason: New leader ebb916052bc64ed597293b7f8e9d6da9. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.826352 32025 sys_catalog.cc:458] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: This master's current role is: FOLLOWER
02:06:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.833184 32028 sys_catalog.cc:455] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.834105 32028 sys_catalog.cc:458] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: This master's current role is: LEADER
02:06:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.836961 32027 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ebb916052bc64ed597293b7f8e9d6da9. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.837728 32027 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: This master's current role is: FOLLOWER
02:06:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.839205 32025 sys_catalog.cc:455] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.840113 32025 sys_catalog.cc:458] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf [sys.catalog]: This master's current role is: FOLLOWER
02:06:30.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:30.840816 32021 catalog_manager.cc:1560] T 00000000000000000000000000000000 P f9dc0c21267e48c3b2b960693e6fbfbf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:30.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 02:06:30.841104 32021 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.848155 32031 catalog_manager.cc:1349] Generated new cluster ID: 93b7bd7bff7f4336b9c14d63fdb361e9
02:06:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.848503 32031 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:30.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.859684 32028 sys_catalog.cc:455] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.860394 32028 sys_catalog.cc:458] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9 [sys.catalog]: This master's current role is: LEADER
02:06:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.865118 32034 mvcc.cc:204] Tried to move back new op lower bound from 7172055002270658560 to 7172055002159005696. Current Snapshot: MvccSnapshot[applied={T|T < 7172055002270658560}]
02:06:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.878391 32031 catalog_manager.cc:1372] Generated new certificate authority record
02:06:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.879604 32027 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ebb916052bc64ed597293b7f8e9d6da9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c0634f06f5a445b08e76834f32aa083f" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 37411 } } peers { permanent_uuid: "ebb916052bc64ed597293b7f8e9d6da9" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 38191 } } peers { permanent_uuid: "f9dc0c21267e48c3b2b960693e6fbfbf" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 40933 } } }
02:06:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.880332 32027 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0634f06f5a445b08e76834f32aa083f [sys.catalog]: This master's current role is: FOLLOWER
02:06:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.880487 32031 catalog_manager.cc:1506] Loading token signing keys...
02:06:30.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.914172 32031 catalog_manager.cc:5955] T 00000000000000000000000000000000 P ebb916052bc64ed597293b7f8e9d6da9: Generated new TSK 0
02:06:30.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 02:06:30.915391 32031 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:30.977 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestTimeouts.testClientNegotiationTimeout: failed attempt 1
java.io.EOFException: null
at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_275]
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)